001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.api.utils;
021
022 import com.google.common.annotations.VisibleForTesting;
023 import com.google.common.base.Joiner;
024 import com.google.common.base.Strings;
025 import com.google.common.collect.ImmutableList;
026 import com.google.common.collect.Lists;
027 import com.google.common.io.ByteStreams;
028 import com.google.common.io.CharStreams;
029 import com.google.common.io.Files;
030 import com.google.common.io.InputSupplier;
031 import org.apache.commons.codec.binary.Base64;
032 import org.apache.commons.io.FileUtils;
033 import org.apache.commons.io.IOUtils;
034 import org.slf4j.LoggerFactory;
035 import org.sonar.api.BatchComponent;
036 import org.sonar.api.ServerComponent;
037 import org.sonar.api.config.Settings;
038 import org.sonar.api.platform.Server;
039
040 import javax.annotation.Nullable;
041
042 import java.io.File;
043 import java.io.IOException;
044 import java.io.InputStream;
045 import java.net.Authenticator;
046 import java.net.HttpURLConnection;
047 import java.net.PasswordAuthentication;
048 import java.net.Proxy;
049 import java.net.ProxySelector;
050 import java.net.URI;
051 import java.nio.charset.Charset;
052 import java.util.List;
053 import java.util.Map;
054 import java.util.zip.GZIPInputStream;
055
056 /**
057 * This component downloads HTTP files
058 *
059 * @since 2.2
060 */
061 public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
062 public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
063
064 private final BaseHttpDownloader downloader;
065 private final Integer readTimeout;
066
067 public HttpDownloader(Server server, Settings settings) {
068 this(server, settings, null);
069 }
070
071 public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) {
072 this.readTimeout = readTimeout;
073 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
074 }
075
076 public HttpDownloader(Settings settings) {
077 this(settings, null);
078 }
079
080 public HttpDownloader(Settings settings, @Nullable Integer readTimeout) {
081 this.readTimeout = readTimeout;
082 downloader = new BaseHttpDownloader(settings.getProperties(), null);
083 }
084
085 @Override
086 String description(URI uri) {
087 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
088 }
089
090 @Override
091 String[] getSupportedSchemes() {
092 return new String[] {"http", "https"};
093 }
094
095 @Override
096 byte[] readBytes(URI uri) {
097 return download(uri);
098 }
099
100 @Override
101 String readString(URI uri, Charset charset) {
102 try {
103 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset));
104 } catch (IOException e) {
105 throw failToDownload(uri, e);
106 }
107 }
108
109 public String downloadPlainText(URI uri, String encoding) {
110 return readString(uri, Charset.forName(encoding));
111 }
112
113 public byte[] download(URI uri) {
114 try {
115 return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout));
116 } catch (IOException e) {
117 throw failToDownload(uri, e);
118 }
119 }
120
121 public String getProxySynthesis(URI uri) {
122 return downloader.getProxySynthesis(uri);
123 }
124
125 public InputStream openStream(URI uri) {
126 try {
127 return downloader.newInputSupplier(uri, this.readTimeout).getInput();
128 } catch (IOException e) {
129 throw failToDownload(uri, e);
130 }
131 }
132
133 public void download(URI uri, File toFile) {
134 try {
135 Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile);
136 } catch (IOException e) {
137 FileUtils.deleteQuietly(toFile);
138 throw failToDownload(uri, e);
139 }
140 }
141
142 private SonarException failToDownload(URI uri, IOException e) {
143 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
144 }
145
146 public static class BaseHttpDownloader {
147
148 private static final String HTTP_PROXY_USER = "http.proxyUser";
149 private static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
150
151 private static final List<String> PROXY_SETTINGS = ImmutableList.of(
152 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
153 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
154
155 private String userAgent;
156
157 public BaseHttpDownloader(Map<String, String> settings, String userAgent) {
158 initProxy(settings);
159 initUserAgent(userAgent);
160 }
161
162 private void initProxy(Map<String, String> settings) {
163 propagateProxySystemProperties(settings);
164 if (requiresProxyAuthentication(settings)) {
165 registerProxyCredentials(settings);
166 }
167 }
168
169 private void initUserAgent(String sonarVersion) {
170 userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion));
171 System.setProperty("http.agent", userAgent);
172 }
173
174 private String getProxySynthesis(URI uri) {
175 return getProxySynthesis(uri, ProxySelector.getDefault());
176 }
177
178 @VisibleForTesting
179 static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
180 List<Proxy> proxies = proxySelector.select(uri);
181 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
182 return "no proxy";
183 }
184
185 List<String> descriptions = Lists.newArrayList();
186 for (Proxy proxy : proxies) {
187 if (proxy.type() != Proxy.Type.DIRECT) {
188 descriptions.add("proxy: " + proxy.address());
189 }
190 }
191
192 return Joiner.on(", ").join(descriptions);
193 }
194
195 private void registerProxyCredentials(Map<String, String> settings) {
196 Authenticator.setDefault(new ProxyAuthenticator(
197 settings.get(HTTP_PROXY_USER),
198 settings.get(HTTP_PROXY_PASSWORD)));
199 }
200
201 private boolean requiresProxyAuthentication(Map<String, String> settings) {
202 return settings.containsKey(HTTP_PROXY_USER);
203 }
204
205 private void propagateProxySystemProperties(Map<String, String> settings) {
206 for (String key : PROXY_SETTINGS) {
207 if (settings.containsKey(key)) {
208 System.setProperty(key, settings.get(key));
209 }
210 }
211 }
212
213 public InputSupplier<InputStream> newInputSupplier(URI uri) {
214 return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
215 }
216
217 public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) {
218 if (readTimeoutMillis != null) {
219 return new HttpInputSupplier(uri, userAgent, null, null, readTimeoutMillis);
220 }
221 return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
222 }
223
224 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
225 return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
226 }
227
228 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) {
229 if (readTimeoutMillis != null) {
230 return new HttpInputSupplier(uri, userAgent, login, password, readTimeoutMillis);
231 }
232 return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
233 }
234
235 private static class HttpInputSupplier implements InputSupplier<InputStream> {
236 private final String login;
237 private final String password;
238 private final URI uri;
239 private final String userAgent;
240 private final int readTimeoutMillis;
241
242 HttpInputSupplier(URI uri, String userAgent, String login, String password, int readTimeoutMillis) {
243 this.uri = uri;
244 this.userAgent = userAgent;
245 this.login = login;
246 this.password = password;
247 this.readTimeoutMillis = readTimeoutMillis;
248 }
249
250 public InputStream getInput() throws IOException {
251 LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
252
253 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
254 HttpsTrust.INSTANCE.trust(connection);
255
256 // allow both GZip and Deflate (ZLib) encodings
257 connection.setRequestProperty("Accept-Encoding", "gzip");
258 if (!Strings.isNullOrEmpty(login)) {
259 String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes()));
260 connection.setRequestProperty("Authorization", "Basic " + encoded);
261 }
262 connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
263 connection.setReadTimeout(readTimeoutMillis);
264 connection.setUseCaches(true);
265 connection.setInstanceFollowRedirects(true);
266 connection.setRequestProperty("User-Agent", userAgent);
267
268 // establish connection, get response headers
269 connection.connect();
270
271 // obtain the encoding returned by the server
272 String encoding = connection.getContentEncoding();
273
274 int responseCode = connection.getResponseCode();
275 if (responseCode >= 400) {
276 InputStream errorResponse = null;
277 try {
278 errorResponse = connection.getErrorStream();
279 if (errorResponse != null) {
280 String errorResponseContent = IOUtils.toString(errorResponse);
281 throw new HttpException(uri, responseCode, errorResponseContent);
282 }
283 throw new HttpException(uri, responseCode);
284
285 } finally {
286 IOUtils.closeQuietly(errorResponse);
287 }
288 }
289
290 InputStream resultingInputStream;
291 // create the appropriate stream wrapper based on the encoding type
292 if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
293 resultingInputStream = new GZIPInputStream(connection.getInputStream());
294 } else {
295 resultingInputStream = connection.getInputStream();
296 }
297 return resultingInputStream;
298 }
299 }
300
301 private static class ProxyAuthenticator extends Authenticator {
302 private final PasswordAuthentication auth;
303
304 ProxyAuthenticator(String user, String password) {
305 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
306 }
307
308 @Override
309 protected PasswordAuthentication getPasswordAuthentication() {
310 return auth;
311 }
312 }
313 }
314
315 public static class HttpException extends RuntimeException {
316 private final URI uri;
317 private final int responseCode;
318 private final String responseContent;
319
320 public HttpException(URI uri, int responseContent) {
321 this(uri, responseContent, null);
322 }
323
324 public HttpException(URI uri, int responseCode, String responseContent) {
325 super("Fail to download [" + uri + "]. Response code: " + responseCode);
326 this.uri = uri;
327 this.responseCode = responseCode;
328 this.responseContent = responseContent;
329 }
330
331 public int getResponseCode() {
332 return responseCode;
333 }
334
335 public URI getUri() {
336 return uri;
337 }
338
339 public String getResponseContent() {
340 return responseContent;
341 }
342 }
343 }