001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.api.utils;
021
022 import com.google.common.annotations.VisibleForTesting;
023 import com.google.common.base.Joiner;
024 import com.google.common.base.Strings;
025 import com.google.common.collect.ImmutableList;
026 import com.google.common.collect.Lists;
027 import com.google.common.io.ByteStreams;
028 import com.google.common.io.CharStreams;
029 import com.google.common.io.Files;
030 import com.google.common.io.InputSupplier;
031 import org.apache.commons.codec.binary.Base64;
032 import org.apache.commons.io.FileUtils;
033 import org.apache.commons.io.IOUtils;
034 import org.slf4j.LoggerFactory;
035 import org.sonar.api.BatchComponent;
036 import org.sonar.api.ServerComponent;
037 import org.sonar.api.config.Settings;
038 import org.sonar.api.platform.Server;
039
040 import javax.annotation.Nullable;
041
042 import java.io.File;
043 import java.io.IOException;
044 import java.io.InputStream;
045 import java.net.Authenticator;
046 import java.net.HttpURLConnection;
047 import java.net.PasswordAuthentication;
048 import java.net.Proxy;
049 import java.net.ProxySelector;
050 import java.net.URI;
051 import java.nio.charset.Charset;
052 import java.util.List;
053 import java.util.Map;
054 import java.util.zip.GZIPInputStream;
055
056 /**
057 * This component downloads HTTP files
058 *
059 * @since 2.2
060 */
061 public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
062 public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
063
064 private final BaseHttpDownloader downloader;
065 private final Integer readTimeout;
066
067 public HttpDownloader(Server server, Settings settings) {
068 this(server, settings, null);
069 }
070
071 public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) {
072 this.readTimeout = readTimeout;
073 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
074 }
075
076 public HttpDownloader(Settings settings) {
077 this(settings, null);
078 }
079
080 public HttpDownloader(Settings settings, @Nullable Integer readTimeout) {
081 this.readTimeout = readTimeout;
082 downloader = new BaseHttpDownloader(settings.getProperties(), null);
083 }
084
085 @Override
086 String description(URI uri) {
087 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
088 }
089
090 @Override
091 String[] getSupportedSchemes() {
092 return new String[] {"http", "https"};
093 }
094
095 @Override
096 byte[] readBytes(URI uri) {
097 return download(uri);
098 }
099
100 @Override
101 String readString(URI uri, Charset charset) {
102 try {
103 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset));
104 } catch (IOException e) {
105 throw failToDownload(uri, e);
106 }
107 }
108
109 public String downloadPlainText(URI uri, String encoding) {
110 return readString(uri, Charset.forName(encoding));
111 }
112
113 public byte[] download(URI uri) {
114 try {
115 return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout));
116 } catch (IOException e) {
117 throw failToDownload(uri, e);
118 }
119 }
120
121 public String getProxySynthesis(URI uri) {
122 return downloader.getProxySynthesis(uri);
123 }
124
125 public InputStream openStream(URI uri) {
126 try {
127 return downloader.newInputSupplier(uri, this.readTimeout).getInput();
128 } catch (IOException e) {
129 throw failToDownload(uri, e);
130 }
131 }
132
133 public void download(URI uri, File toFile) {
134 try {
135 Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile);
136 } catch (IOException e) {
137 FileUtils.deleteQuietly(toFile);
138 throw failToDownload(uri, e);
139 }
140 }
141
142 private SonarException failToDownload(URI uri, IOException e) {
143 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
144 }
145
146 public static class BaseHttpDownloader {
147
148 private static final String GET = "GET";
149 private static final String HTTP_PROXY_USER = "http.proxyUser";
150 private static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
151
152 private static final List<String> PROXY_SETTINGS = ImmutableList.of(
153 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
154 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
155
156 private String userAgent;
157
158 public BaseHttpDownloader(Map<String, String> settings, @Nullable String userAgent) {
159 initProxy(settings);
160 initUserAgent(userAgent);
161 }
162
163 private void initProxy(Map<String, String> settings) {
164 propagateProxySystemProperties(settings);
165 if (requiresProxyAuthentication(settings)) {
166 registerProxyCredentials(settings);
167 }
168 }
169
170 private void initUserAgent(@Nullable String sonarVersion) {
171 userAgent = (sonarVersion == null ? "SonarQube" : String.format("SonarQube %s", sonarVersion));
172 System.setProperty("http.agent", userAgent);
173 }
174
175 private String getProxySynthesis(URI uri) {
176 return getProxySynthesis(uri, ProxySelector.getDefault());
177 }
178
179 @VisibleForTesting
180 static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
181 List<Proxy> proxies = proxySelector.select(uri);
182 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
183 return "no proxy";
184 }
185
186 List<String> descriptions = Lists.newArrayList();
187 for (Proxy proxy : proxies) {
188 if (proxy.type() != Proxy.Type.DIRECT) {
189 descriptions.add(proxy.type() + " proxy: " + proxy.address());
190 }
191 }
192
193 return Joiner.on(", ").join(descriptions);
194 }
195
196 private void registerProxyCredentials(Map<String, String> settings) {
197 Authenticator.setDefault(new ProxyAuthenticator(
198 settings.get(HTTP_PROXY_USER),
199 settings.get(HTTP_PROXY_PASSWORD)));
200 }
201
202 private boolean requiresProxyAuthentication(Map<String, String> settings) {
203 return settings.containsKey(HTTP_PROXY_USER);
204 }
205
206 private void propagateProxySystemProperties(Map<String, String> settings) {
207 for (String key : PROXY_SETTINGS) {
208 if (settings.containsKey(key)) {
209 System.setProperty(key, settings.get(key));
210 }
211 }
212 }
213
214 public InputSupplier<InputStream> newInputSupplier(URI uri) {
215 return new HttpInputSupplier(uri, GET, userAgent, null, null, TIMEOUT_MILLISECONDS);
216 }
217
218 public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) {
219 return newInputSupplier(uri, GET, readTimeoutMillis);
220 }
221
222 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, @Nullable Integer readTimeoutMillis) {
223 if (readTimeoutMillis != null) {
224 return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, readTimeoutMillis);
225 }
226 return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, TIMEOUT_MILLISECONDS);
227 }
228
229 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
230 return newInputSupplier(uri, GET, login, password);
231 }
232
233 /**
234 * @since 5.0
235 */
236 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password) {
237 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS);
238 }
239
240 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) {
241 return newInputSupplier(uri, GET, login, password, readTimeoutMillis);
242 }
243
244 /**
245 * @since 5.0
246 */
247 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password, @Nullable Integer readTimeoutMillis) {
248 if (readTimeoutMillis != null) {
249 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, readTimeoutMillis);
250 }
251 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS);
252 }
253
254 private static class HttpInputSupplier implements InputSupplier<InputStream> {
255 private final String login;
256 private final String password;
257 private final URI uri;
258 private final String userAgent;
259 private final int readTimeoutMillis;
260 private final String requestMethod;
261
262 HttpInputSupplier(URI uri, String requestMethod, String userAgent, String login, String password, int readTimeoutMillis) {
263 this.uri = uri;
264 this.requestMethod = requestMethod;
265 this.userAgent = userAgent;
266 this.login = login;
267 this.password = password;
268 this.readTimeoutMillis = readTimeoutMillis;
269 }
270
271 @Override
272 public InputStream getInput() throws IOException {
273 LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
274
275 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
276 connection.setRequestMethod(requestMethod);
277 HttpsTrust.INSTANCE.trust(connection);
278
279 // allow both GZip and Deflate (ZLib) encodings
280 connection.setRequestProperty("Accept-Encoding", "gzip");
281 if (!Strings.isNullOrEmpty(login)) {
282 String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes()));
283 connection.setRequestProperty("Authorization", "Basic " + encoded);
284 }
285 connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
286 connection.setReadTimeout(readTimeoutMillis);
287 connection.setUseCaches(true);
288 connection.setInstanceFollowRedirects(true);
289 connection.setRequestProperty("User-Agent", userAgent);
290
291 // establish connection, get response headers
292 connection.connect();
293
294 // obtain the encoding returned by the server
295 String encoding = connection.getContentEncoding();
296
297 int responseCode = connection.getResponseCode();
298 if (responseCode >= 400) {
299 InputStream errorResponse = null;
300 try {
301 errorResponse = connection.getErrorStream();
302 if (errorResponse != null) {
303 String errorResponseContent = IOUtils.toString(errorResponse);
304 throw new HttpException(uri, responseCode, errorResponseContent);
305 }
306 throw new HttpException(uri, responseCode);
307
308 } finally {
309 IOUtils.closeQuietly(errorResponse);
310 }
311 }
312
313 InputStream resultingInputStream;
314 // create the appropriate stream wrapper based on the encoding type
315 if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
316 resultingInputStream = new GZIPInputStream(connection.getInputStream());
317 } else {
318 resultingInputStream = connection.getInputStream();
319 }
320 return resultingInputStream;
321 }
322 }
323
324 private static class ProxyAuthenticator extends Authenticator {
325 private final PasswordAuthentication auth;
326
327 ProxyAuthenticator(String user, String password) {
328 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
329 }
330
331 @Override
332 protected PasswordAuthentication getPasswordAuthentication() {
333 return auth;
334 }
335 }
336 }
337
338 public static class HttpException extends RuntimeException {
339 private final URI uri;
340 private final int responseCode;
341 private final String responseContent;
342
343 public HttpException(URI uri, int responseContent) {
344 this(uri, responseContent, "");
345 }
346
347 public HttpException(URI uri, int responseCode, String responseContent) {
348 super("Fail to download [" + uri + "]. Response code: " + responseCode);
349 this.uri = uri;
350 this.responseCode = responseCode;
351 this.responseContent = responseContent;
352 }
353
354 public int getResponseCode() {
355 return responseCode;
356 }
357
358 public URI getUri() {
359 return uri;
360 }
361
362 public String getResponseContent() {
363 return responseContent;
364 }
365 }
366 }