001/*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
019 */
020package org.sonar.api.utils;
021
022import com.google.common.annotations.VisibleForTesting;
023import com.google.common.base.Joiner;
024import com.google.common.base.Strings;
025import com.google.common.collect.ImmutableList;
026import com.google.common.collect.Lists;
027import com.google.common.io.ByteStreams;
028import com.google.common.io.CharStreams;
029import com.google.common.io.Files;
030import com.google.common.io.InputSupplier;
031import org.apache.commons.codec.binary.Base64;
032import org.apache.commons.io.Charsets;
033import org.apache.commons.io.FileUtils;
034import org.apache.commons.io.IOUtils;
035import org.sonar.api.BatchComponent;
036import org.sonar.api.ServerComponent;
037import org.sonar.api.config.Settings;
038import org.sonar.api.platform.Server;
039import org.sonar.api.utils.log.Loggers;
040
041import javax.annotation.Nullable;
042
043import java.io.File;
044import java.io.IOException;
045import java.io.InputStream;
046import java.net.Authenticator;
047import java.net.HttpURLConnection;
048import java.net.PasswordAuthentication;
049import java.net.Proxy;
050import java.net.ProxySelector;
051import java.net.URI;
052import java.nio.charset.Charset;
053import java.util.List;
054import java.util.Map;
055import java.util.zip.GZIPInputStream;
056
057/**
058 * This component downloads HTTP files
059 *
060 * @since 2.2
061 */
062public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
063  public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
064
065  private final BaseHttpDownloader downloader;
066  private final Integer readTimeout;
067
068  public HttpDownloader(Server server, Settings settings) {
069    this(server, settings, null);
070  }
071
072  public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) {
073    this.readTimeout = readTimeout;
074    downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
075  }
076
077  public HttpDownloader(Settings settings) {
078    this(settings, null);
079  }
080
081  public HttpDownloader(Settings settings, @Nullable Integer readTimeout) {
082    this.readTimeout = readTimeout;
083    downloader = new BaseHttpDownloader(settings.getProperties(), null);
084  }
085
086  @Override
087  String description(URI uri) {
088    return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
089  }
090
091  @Override
092  String[] getSupportedSchemes() {
093    return new String[] {"http", "https"};
094  }
095
096  @Override
097  byte[] readBytes(URI uri) {
098    return download(uri);
099  }
100
101  @Override
102  String readString(URI uri, Charset charset) {
103    try {
104      return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset));
105    } catch (IOException e) {
106      throw failToDownload(uri, e);
107    }
108  }
109
110  public String downloadPlainText(URI uri, String encoding) {
111    return readString(uri, Charset.forName(encoding));
112  }
113
114  public byte[] download(URI uri) {
115    try {
116      return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout));
117    } catch (IOException e) {
118      throw failToDownload(uri, e);
119    }
120  }
121
122  public String getProxySynthesis(URI uri) {
123    return downloader.getProxySynthesis(uri);
124  }
125
126  public InputStream openStream(URI uri) {
127    try {
128      return downloader.newInputSupplier(uri, this.readTimeout).getInput();
129    } catch (IOException e) {
130      throw failToDownload(uri, e);
131    }
132  }
133
134  public void download(URI uri, File toFile) {
135    try {
136      Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile);
137    } catch (IOException e) {
138      FileUtils.deleteQuietly(toFile);
139      throw failToDownload(uri, e);
140    }
141  }
142
143  private SonarException failToDownload(URI uri, IOException e) {
144    throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
145  }
146
147  public static class BaseHttpDownloader {
148
149    private static final String GET = "GET";
150    private static final String HTTP_PROXY_USER = "http.proxyUser";
151    private static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
152
153    private static final List<String> PROXY_SETTINGS = ImmutableList.of(
154      "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
155      "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
156
157    private String userAgent;
158
159    public BaseHttpDownloader(Map<String, String> settings, @Nullable String userAgent) {
160      initProxy(settings);
161      initUserAgent(userAgent);
162    }
163
164    private void initProxy(Map<String, String> settings) {
165      propagateProxySystemProperties(settings);
166      if (requiresProxyAuthentication(settings)) {
167        registerProxyCredentials(settings);
168      }
169    }
170
171    private void initUserAgent(@Nullable String sonarVersion) {
172      userAgent = (sonarVersion == null ? "SonarQube" : String.format("SonarQube %s", sonarVersion));
173      System.setProperty("http.agent", userAgent);
174    }
175
176    private String getProxySynthesis(URI uri) {
177      return getProxySynthesis(uri, ProxySelector.getDefault());
178    }
179
180    @VisibleForTesting
181    static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
182      List<Proxy> proxies = proxySelector.select(uri);
183      if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
184        return "no proxy";
185      }
186
187      List<String> descriptions = Lists.newArrayList();
188      for (Proxy proxy : proxies) {
189        if (proxy.type() != Proxy.Type.DIRECT) {
190          descriptions.add(proxy.type() + " proxy: " + proxy.address());
191        }
192      }
193
194      return Joiner.on(", ").join(descriptions);
195    }
196
197    private void registerProxyCredentials(Map<String, String> settings) {
198      Authenticator.setDefault(new ProxyAuthenticator(
199        settings.get(HTTP_PROXY_USER),
200        settings.get(HTTP_PROXY_PASSWORD)));
201    }
202
203    private boolean requiresProxyAuthentication(Map<String, String> settings) {
204      return settings.containsKey(HTTP_PROXY_USER);
205    }
206
207    private void propagateProxySystemProperties(Map<String, String> settings) {
208      for (String key : PROXY_SETTINGS) {
209        if (settings.containsKey(key)) {
210          System.setProperty(key, settings.get(key));
211        }
212      }
213    }
214
215    public InputSupplier<InputStream> newInputSupplier(URI uri) {
216      return new HttpInputSupplier(uri, GET, userAgent, null, null, TIMEOUT_MILLISECONDS);
217    }
218
219    public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) {
220      return newInputSupplier(uri, GET, readTimeoutMillis);
221    }
222
223    public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, @Nullable Integer readTimeoutMillis) {
224      if (readTimeoutMillis != null) {
225        return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, readTimeoutMillis);
226      }
227      return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, TIMEOUT_MILLISECONDS);
228    }
229
230    public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
231      return newInputSupplier(uri, GET, login, password);
232    }
233
234    /**
235     * @since 5.0
236     */
237    public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password) {
238      return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS);
239    }
240
241    public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) {
242      return newInputSupplier(uri, GET, login, password, readTimeoutMillis);
243    }
244
245    /**
246     * @since 5.0
247     */
248    public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password, @Nullable Integer readTimeoutMillis) {
249      if (readTimeoutMillis != null) {
250        return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, readTimeoutMillis);
251      }
252      return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS);
253    }
254
255    private static class HttpInputSupplier implements InputSupplier<InputStream> {
256      private final String login;
257      private final String password;
258      private final URI uri;
259      private final String userAgent;
260      private final int readTimeoutMillis;
261      private final String requestMethod;
262
263      HttpInputSupplier(URI uri, String requestMethod, String userAgent, String login, String password, int readTimeoutMillis) {
264        this.uri = uri;
265        this.requestMethod = requestMethod;
266        this.userAgent = userAgent;
267        this.login = login;
268        this.password = password;
269        this.readTimeoutMillis = readTimeoutMillis;
270      }
271
272      @Override
273      public InputStream getInput() throws IOException {
274        Loggers.get(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
275
276        HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
277        connection.setRequestMethod(requestMethod);
278        HttpsTrust.INSTANCE.trust(connection);
279
280        // allow both GZip and Deflate (ZLib) encodings
281        connection.setRequestProperty("Accept-Encoding", "gzip");
282        if (!Strings.isNullOrEmpty(login)) {
283          String encoded = Base64.encodeBase64String((login + ":" + password).getBytes(Charsets.UTF_8));
284          connection.setRequestProperty("Authorization", "Basic " + encoded);
285        }
286        connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
287        connection.setReadTimeout(readTimeoutMillis);
288        connection.setUseCaches(true);
289        connection.setInstanceFollowRedirects(true);
290        connection.setRequestProperty("User-Agent", userAgent);
291
292        // establish connection, get response headers
293        connection.connect();
294
295        // obtain the encoding returned by the server
296        String encoding = connection.getContentEncoding();
297
298        int responseCode = connection.getResponseCode();
299        if (responseCode >= 400) {
300          InputStream errorResponse = null;
301          try {
302            errorResponse = connection.getErrorStream();
303            if (errorResponse != null) {
304              String errorResponseContent = IOUtils.toString(errorResponse);
305              throw new HttpException(uri, responseCode, errorResponseContent);
306            }
307            throw new HttpException(uri, responseCode);
308
309          } finally {
310            IOUtils.closeQuietly(errorResponse);
311          }
312        }
313
314        InputStream resultingInputStream;
315        // create the appropriate stream wrapper based on the encoding type
316        if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
317          resultingInputStream = new GZIPInputStream(connection.getInputStream());
318        } else {
319          resultingInputStream = connection.getInputStream();
320        }
321        return resultingInputStream;
322      }
323    }
324
325    private static class ProxyAuthenticator extends Authenticator {
326      private final PasswordAuthentication auth;
327
328      ProxyAuthenticator(String user, String password) {
329        auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
330      }
331
332      @Override
333      protected PasswordAuthentication getPasswordAuthentication() {
334        return auth;
335      }
336    }
337  }
338
339  public static class HttpException extends RuntimeException {
340    private final URI uri;
341    private final int responseCode;
342    private final String responseContent;
343
344    public HttpException(URI uri, int responseContent) {
345      this(uri, responseContent, "");
346    }
347
348    public HttpException(URI uri, int responseCode, String responseContent) {
349      super("Fail to download [" + uri + "]. Response code: " + responseCode);
350      this.uri = uri;
351      this.responseCode = responseCode;
352      this.responseContent = responseContent;
353    }
354
355    public int getResponseCode() {
356      return responseCode;
357    }
358
359    public URI getUri() {
360      return uri;
361    }
362
363    public String getResponseContent() {
364      return responseContent;
365    }
366  }
367}