001/*
002 * Sonar, open source software quality management tool.
003 * Copyright (C) 2008-2012 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * Sonar is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * Sonar is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public
017 * License along with Sonar; if not, write to the Free Software
018 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
019 */
020package org.sonar.api.utils;
021
022import com.google.common.annotations.VisibleForTesting;
023import com.google.common.base.Joiner;
024import com.google.common.base.Strings;
025import com.google.common.collect.ImmutableList;
026import com.google.common.collect.Lists;
027import com.google.common.io.ByteStreams;
028import com.google.common.io.CharStreams;
029import com.google.common.io.Files;
030import com.google.common.io.InputSupplier;
031import org.apache.commons.codec.binary.Base64;
032import org.apache.commons.io.FileUtils;
033import org.slf4j.LoggerFactory;
034import org.sonar.api.BatchComponent;
035import org.sonar.api.ServerComponent;
036import org.sonar.api.config.Settings;
037import org.sonar.api.platform.Server;
038
039import java.io.File;
040import java.io.IOException;
041import java.io.InputStream;
042import java.net.Authenticator;
043import java.net.HttpURLConnection;
044import java.net.PasswordAuthentication;
045import java.net.Proxy;
046import java.net.ProxySelector;
047import java.net.URI;
048import java.nio.charset.Charset;
049import java.util.List;
050import java.util.Map;
051
052/**
053 * This component downloads HTTP files
054 *
055 * @since 2.2
056 */
057public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
058  public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
059
060  private final BaseHttpDownloader downloader;
061
062  public HttpDownloader(Server server, Settings settings) {
063    downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
064  }
065
066  public HttpDownloader(Settings settings) {
067    downloader = new BaseHttpDownloader(settings.getProperties(), null);
068  }
069
070  @Override
071  String description(URI uri) {
072    return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
073  }
074
075  @Override
076  String[] getSupportedSchemes() {
077    return new String[]{"http", "https"};
078  }
079
080  @Override
081  byte[] readBytes(URI uri) {
082    return download(uri);
083  }
084
085  @Override
086  String readString(URI uri, Charset charset) {
087    try {
088      return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri), charset));
089    } catch (IOException e) {
090      throw failToDownload(uri, e);
091    }
092  }
093
094  public String downloadPlainText(URI uri, String encoding) {
095    return readString(uri, Charset.forName(encoding));
096  }
097
098  public byte[] download(URI uri) {
099    try {
100      return ByteStreams.toByteArray(downloader.newInputSupplier(uri));
101    } catch (IOException e) {
102      throw failToDownload(uri, e);
103    }
104  }
105
106  public String getProxySynthesis(URI uri) {
107    return downloader.getProxySynthesis(uri);
108  }
109
110  public InputStream openStream(URI uri) {
111    try {
112      return downloader.newInputSupplier(uri).getInput();
113    } catch (IOException e) {
114      throw failToDownload(uri, e);
115    }
116  }
117
118  public void download(URI uri, File toFile) {
119    try {
120      Files.copy(downloader.newInputSupplier(uri), toFile);
121    } catch (IOException e) {
122      FileUtils.deleteQuietly(toFile);
123      throw failToDownload(uri, e);
124    }
125  }
126
127  private SonarException failToDownload(URI uri, IOException e) {
128    throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
129  }
130
131  public static class BaseHttpDownloader {
132    private static final List<String> PROXY_SETTINGS = ImmutableList.of(
133      "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
134      "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
135
136    private String userAgent;
137
138    public BaseHttpDownloader(Map<String, String> settings, String userAgent) {
139      initProxy(settings);
140      initUserAgent(userAgent);
141    }
142
143    private void initProxy(Map<String, String> settings) {
144      propagateProxySystemProperties(settings);
145      if (requiresProxyAuthentication(settings)) {
146        registerProxyCredentials(settings);
147      }
148    }
149
150    private void initUserAgent(String sonarVersion) {
151      userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion));
152      System.setProperty("http.agent", userAgent);
153    }
154
155    private String getProxySynthesis(URI uri) {
156      return getProxySynthesis(uri, ProxySelector.getDefault());
157    }
158
159    @VisibleForTesting
160    static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
161      List<Proxy> proxies = proxySelector.select(uri);
162      if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
163        return "no proxy";
164      }
165
166      List<String> descriptions = Lists.newArrayList();
167      for (Proxy proxy : proxies) {
168        if (proxy.type() != Proxy.Type.DIRECT) {
169          descriptions.add("proxy: " + proxy.address());
170        }
171      }
172
173      return Joiner.on(", ").join(descriptions);
174    }
175
176    private void registerProxyCredentials(Map<String, String> settings) {
177      Authenticator.setDefault(new ProxyAuthenticator(
178        settings.get("http.proxyUser"),
179        settings.get("http.proxyPassword")));
180    }
181
182    private boolean requiresProxyAuthentication(Map<String, String> settings) {
183      return settings.containsKey("http.proxyUser");
184    }
185
186    private void propagateProxySystemProperties(Map<String, String> settings) {
187      for (String key : PROXY_SETTINGS) {
188        if (settings.containsKey(key)) {
189          System.setProperty(key, settings.get(key));
190        }
191      }
192    }
193
194    public InputSupplier<InputStream> newInputSupplier(URI uri) {
195      return new HttpInputSupplier(uri, userAgent, null, null);
196    }
197
198    public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
199      return new HttpInputSupplier(uri, userAgent, login, password);
200    }
201
202    private static class HttpInputSupplier implements InputSupplier<InputStream> {
203      private final String login;
204      private final String password;
205      private final URI uri;
206      private final String userAgent;
207
208      HttpInputSupplier(URI uri, String userAgent, String login, String password) {
209        this.uri = uri;
210        this.userAgent = userAgent;
211        this.login = login;
212        this.password = password;
213      }
214
215      public InputStream getInput() throws IOException {
216        LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
217
218        HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
219        if (!Strings.isNullOrEmpty(login)) {
220          String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes()));
221          connection.setRequestProperty("Authorization", "Basic " + encoded);
222        }
223        connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
224        connection.setReadTimeout(TIMEOUT_MILLISECONDS);
225        connection.setUseCaches(true);
226        connection.setInstanceFollowRedirects(true);
227        connection.setRequestProperty("User-Agent", userAgent);
228
229        int responseCode = connection.getResponseCode();
230        if (responseCode >= 400) {
231          throw new HttpException(uri, responseCode);
232        }
233
234        return connection.getInputStream();
235      }
236    }
237
238    private static class ProxyAuthenticator extends Authenticator {
239      private final PasswordAuthentication auth;
240
241      ProxyAuthenticator(String user, String password) {
242        auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
243      }
244
245      @Override
246      protected PasswordAuthentication getPasswordAuthentication() {
247        return auth;
248      }
249    }
250  }
251
252  public static class HttpException extends RuntimeException {
253    private final URI uri;
254    private final int responseCode;
255
256    public HttpException(URI uri, int responseCode) {
257      super("Fail to download [" + uri + "]. Response code: " + responseCode);
258      this.uri = uri;
259      this.responseCode = responseCode;
260    }
261
262    public int getResponseCode() {
263      return responseCode;
264    }
265
266    public URI getUri() {
267      return uri;
268    }
269  }
270}