Source code

001/*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2013 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
019 */
020package org.sonar.api.utils;
021
022import com.google.common.annotations.VisibleForTesting;
023import com.google.common.base.Joiner;
024import com.google.common.base.Strings;
025import com.google.common.collect.ImmutableList;
026import com.google.common.collect.Lists;
027import com.google.common.io.ByteStreams;
028import com.google.common.io.CharStreams;
029import com.google.common.io.Files;
030import com.google.common.io.InputSupplier;
031import org.apache.commons.codec.binary.Base64;
032import org.apache.commons.io.FileUtils;
033import org.apache.commons.io.IOUtils;
034import org.slf4j.LoggerFactory;
035import org.sonar.api.BatchComponent;
036import org.sonar.api.ServerComponent;
037import org.sonar.api.config.Settings;
038import org.sonar.api.platform.Server;
039
040import javax.annotation.Nullable;
041
042import java.io.File;
043import java.io.IOException;
044import java.io.InputStream;
045import java.net.Authenticator;
046import java.net.HttpURLConnection;
047import java.net.PasswordAuthentication;
048import java.net.Proxy;
049import java.net.ProxySelector;
050import java.net.URI;
051import java.nio.charset.Charset;
052import java.util.List;
053import java.util.Map;
054import java.util.zip.GZIPInputStream;
055
056/**
057 * This component downloads HTTP files
058 *
059 * @since 2.2
060 */
061public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
062  public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
063
064  private final BaseHttpDownloader downloader;
065  private final Integer readTimeout;
066
067  public HttpDownloader(Server server, Settings settings) {
068    this(server, settings, null);
069  }
070
071  public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) {
072    this.readTimeout = readTimeout;
073    downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
074  }
075
076  public HttpDownloader(Settings settings) {
077    this(settings, null);
078  }
079
080  public HttpDownloader(Settings settings, @Nullable Integer readTimeout) {
081    this.readTimeout = readTimeout;
082    downloader = new BaseHttpDownloader(settings.getProperties(), null);
083  }
084
085  @Override
086  String description(URI uri) {
087    return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
088  }
089
090  @Override
091  String[] getSupportedSchemes() {
092    return new String[] {"http", "https"};
093  }
094
095  @Override
096  byte[] readBytes(URI uri) {
097    return download(uri);
098  }
099
100  @Override
101  String readString(URI uri, Charset charset) {
102    try {
103      return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset));
104    } catch (IOException e) {
105      throw failToDownload(uri, e);
106    }
107  }
108
109  public String downloadPlainText(URI uri, String encoding) {
110    return readString(uri, Charset.forName(encoding));
111  }
112
113  public byte[] download(URI uri) {
114    try {
115      return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout));
116    } catch (IOException e) {
117      throw failToDownload(uri, e);
118    }
119  }
120
121  public String getProxySynthesis(URI uri) {
122    return downloader.getProxySynthesis(uri);
123  }
124
125  public InputStream openStream(URI uri) {
126    try {
127      return downloader.newInputSupplier(uri, this.readTimeout).getInput();
128    } catch (IOException e) {
129      throw failToDownload(uri, e);
130    }
131  }
132
133  public void download(URI uri, File toFile) {
134    try {
135      Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile);
136    } catch (IOException e) {
137      FileUtils.deleteQuietly(toFile);
138      throw failToDownload(uri, e);
139    }
140  }
141
142  private SonarException failToDownload(URI uri, IOException e) {
143    throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
144  }
145
146  public static class BaseHttpDownloader {
147
148    private static final String HTTP_PROXY_USER = "http.proxyUser";
149    private static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
150
151    private static final List<String> PROXY_SETTINGS = ImmutableList.of(
152      "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
153      "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
154
155    private String userAgent;
156
157    public BaseHttpDownloader(Map<String, String> settings, String userAgent) {
158      initProxy(settings);
159      initUserAgent(userAgent);
160    }
161
162    private void initProxy(Map<String, String> settings) {
163      propagateProxySystemProperties(settings);
164      if (requiresProxyAuthentication(settings)) {
165        registerProxyCredentials(settings);
166      }
167    }
168
169    private void initUserAgent(String sonarVersion) {
170      userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion));
171      System.setProperty("http.agent", userAgent);
172    }
173
174    private String getProxySynthesis(URI uri) {
175      return getProxySynthesis(uri, ProxySelector.getDefault());
176    }
177
178    @VisibleForTesting
179    static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
180      List<Proxy> proxies = proxySelector.select(uri);
181      if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
182        return "no proxy";
183      }
184
185      List<String> descriptions = Lists.newArrayList();
186      for (Proxy proxy : proxies) {
187        if (proxy.type() != Proxy.Type.DIRECT) {
188          descriptions.add("proxy: " + proxy.address());
189        }
190      }
191
192      return Joiner.on(", ").join(descriptions);
193    }
194
195    private void registerProxyCredentials(Map<String, String> settings) {
196      Authenticator.setDefault(new ProxyAuthenticator(
197        settings.get(HTTP_PROXY_USER),
198        settings.get(HTTP_PROXY_PASSWORD)));
199    }
200
201    private boolean requiresProxyAuthentication(Map<String, String> settings) {
202      return settings.containsKey(HTTP_PROXY_USER);
203    }
204
205    private void propagateProxySystemProperties(Map<String, String> settings) {
206      for (String key : PROXY_SETTINGS) {
207        if (settings.containsKey(key)) {
208          System.setProperty(key, settings.get(key));
209        }
210      }
211    }
212
213    public InputSupplier<InputStream> newInputSupplier(URI uri) {
214      return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
215    }
216
217    public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) {
218      if (readTimeoutMillis != null) {
219        return new HttpInputSupplier(uri, userAgent, null, null, readTimeoutMillis);
220      }
221      return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
222    }
223
224    public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
225      return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
226    }
227
228    public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) {
229      if (readTimeoutMillis != null) {
230        return new HttpInputSupplier(uri, userAgent, login, password, readTimeoutMillis);
231      }
232      return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
233    }
234
235    private static class HttpInputSupplier implements InputSupplier<InputStream> {
236      private final String login;
237      private final String password;
238      private final URI uri;
239      private final String userAgent;
240      private final int readTimeoutMillis;
241
242      HttpInputSupplier(URI uri, String userAgent, String login, String password, int readTimeoutMillis) {
243        this.uri = uri;
244        this.userAgent = userAgent;
245        this.login = login;
246        this.password = password;
247        this.readTimeoutMillis = readTimeoutMillis;
248      }
249
250      public InputStream getInput() throws IOException {
251        LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
252
253        HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
254        HttpsTrust.INSTANCE.trust(connection);
255
256        // allow both GZip and Deflate (ZLib) encodings
257        connection.setRequestProperty("Accept-Encoding", "gzip");
258        if (!Strings.isNullOrEmpty(login)) {
259          String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes()));
260          connection.setRequestProperty("Authorization", "Basic " + encoded);
261        }
262        connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
263        connection.setReadTimeout(readTimeoutMillis);
264        connection.setUseCaches(true);
265        connection.setInstanceFollowRedirects(true);
266        connection.setRequestProperty("User-Agent", userAgent);
267
268        // establish connection, get response headers
269        connection.connect();
270
271        // obtain the encoding returned by the server
272        String encoding = connection.getContentEncoding();
273
274        int responseCode = connection.getResponseCode();
275        if (responseCode >= 400) {
276          InputStream errorResponse = null;
277          try {
278            errorResponse = connection.getErrorStream();
279            if (errorResponse != null) {
280              String errorResponseContent = IOUtils.toString(errorResponse);
281              throw new HttpException(uri, responseCode, errorResponseContent);
282            }
283            throw new HttpException(uri, responseCode);
284
285          } finally {
286            IOUtils.closeQuietly(errorResponse);
287          }
288        }
289
290        InputStream resultingInputStream = null;
291        // create the appropriate stream wrapper based on the encoding type
292        if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
293          resultingInputStream = new GZIPInputStream(connection.getInputStream());
294        } else {
295          resultingInputStream = connection.getInputStream();
296        }
297        return resultingInputStream;
298      }
299    }
300
301    private static class ProxyAuthenticator extends Authenticator {
302      private final PasswordAuthentication auth;
303
304      ProxyAuthenticator(String user, String password) {
305        auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
306      }
307
308      @Override
309      protected PasswordAuthentication getPasswordAuthentication() {
310        return auth;
311      }
312    }
313  }
314
315  public static class HttpException extends RuntimeException {
316    private final URI uri;
317    private final int responseCode;
318    private final String responseContent;
319
320    public HttpException(URI uri, int responseContent) {
321      this(uri, responseContent, null);
322    }
323
324    public HttpException(URI uri, int responseCode, String responseContent) {
325      super("Fail to download [" + uri + "]. Response code: " + responseCode);
326      this.uri = uri;
327      this.responseCode = responseCode;
328      this.responseContent = responseContent;
329    }
330
331    public int getResponseCode() {
332      return responseCode;
333    }
334
335    public URI getUri() {
336      return uri;
337    }
338
339    public String getResponseContent() {
340      return responseContent;
341    }
342  }
343}