001/* 002 * SonarQube, open source software quality management tool. 003 * Copyright (C) 2008-2014 SonarSource 004 * mailto:contact AT sonarsource DOT com 005 * 006 * SonarQube is free software; you can redistribute it and/or 007 * modify it under the terms of the GNU Lesser General Public 008 * License as published by the Free Software Foundation; either 009 * version 3 of the License, or (at your option) any later version. 010 * 011 * SonarQube is distributed in the hope that it will be useful, 012 * but WITHOUT ANY WARRANTY; without even the implied warranty of 013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 014 * Lesser General Public License for more details. 015 * 016 * You should have received a copy of the GNU Lesser General Public License 017 * along with this program; if not, write to the Free Software Foundation, 018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 019 */ 020package org.sonar.api.utils; 021 022import com.google.common.annotations.VisibleForTesting; 023import com.google.common.base.Joiner; 024import com.google.common.base.Strings; 025import com.google.common.collect.ImmutableList; 026import com.google.common.collect.Lists; 027import com.google.common.io.ByteStreams; 028import com.google.common.io.CharStreams; 029import com.google.common.io.Files; 030import com.google.common.io.InputSupplier; 031import org.apache.commons.codec.binary.Base64; 032import org.apache.commons.io.Charsets; 033import org.apache.commons.io.FileUtils; 034import org.apache.commons.io.IOUtils; 035import org.sonar.api.BatchComponent; 036import org.sonar.api.ServerComponent; 037import org.sonar.api.config.Settings; 038import org.sonar.api.platform.Server; 039import org.sonar.api.utils.log.Loggers; 040 041import javax.annotation.Nullable; 042 043import java.io.File; 044import java.io.IOException; 045import java.io.InputStream; 046import java.net.Authenticator; 047import java.net.HttpURLConnection; 048import java.net.PasswordAuthentication; 049import java.net.Proxy; 050import java.net.ProxySelector; 051import java.net.URI; 052import java.nio.charset.Charset; 053import java.util.List; 054import java.util.Map; 055import java.util.zip.GZIPInputStream; 056 057/** 058 * This component downloads HTTP files 059 * 060 * @since 2.2 061 */ 062public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent { 063 public static final int TIMEOUT_MILLISECONDS = 20 * 1000; 064 065 private final BaseHttpDownloader downloader; 066 private final Integer readTimeout; 067 068 public HttpDownloader(Server server, Settings settings) { 069 this(server, settings, null); 070 } 071 072 public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) { 073 this.readTimeout = readTimeout; 074 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion()); 075 } 076 077 public HttpDownloader(Settings settings) { 078 this(settings, null); 079 } 080 081 public HttpDownloader(Settings settings, @Nullable Integer readTimeout) { 082 this.readTimeout = readTimeout; 083 downloader = new BaseHttpDownloader(settings.getProperties(), null); 084 } 085 086 @Override 087 String description(URI uri) { 088 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri)); 089 } 090 091 @Override 092 String[] getSupportedSchemes() { 093 return new String[] {"http", "https"}; 094 } 095 096 @Override 097 byte[] readBytes(URI uri) { 098 return download(uri); 099 } 100 101 @Override 102 String readString(URI uri, Charset charset) { 103 try { 104 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset)); 105 } catch (IOException e) { 106 throw failToDownload(uri, e); 107 } 108 } 109 110 public String downloadPlainText(URI uri, String encoding) { 111 return readString(uri, Charset.forName(encoding)); 112 } 113 114 public byte[] download(URI uri) { 115 try { 116 return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout)); 117 } catch (IOException e) { 118 throw failToDownload(uri, e); 119 } 120 } 121 122 public String getProxySynthesis(URI uri) { 123 return downloader.getProxySynthesis(uri); 124 } 125 126 public InputStream openStream(URI uri) { 127 try { 128 return downloader.newInputSupplier(uri, this.readTimeout).getInput(); 129 } catch (IOException e) { 130 throw failToDownload(uri, e); 131 } 132 } 133 134 public void download(URI uri, File toFile) { 135 try { 136 Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile); 137 } catch (IOException e) { 138 FileUtils.deleteQuietly(toFile); 139 throw failToDownload(uri, e); 140 } 141 } 142 143 private SonarException failToDownload(URI uri, IOException e) { 144 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e); 145 } 146 147 public static class BaseHttpDownloader { 148 149 private static final String GET = "GET"; 150 private static final String HTTP_PROXY_USER = "http.proxyUser"; 151 private static final String HTTP_PROXY_PASSWORD = "http.proxyPassword"; 152 153 private static final List<String> PROXY_SETTINGS = ImmutableList.of( 154 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts", 155 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort"); 156 157 private String userAgent; 158 159 public BaseHttpDownloader(Map<String, String> settings, @Nullable String userAgent) { 160 initProxy(settings); 161 initUserAgent(userAgent); 162 } 163 164 private void initProxy(Map<String, String> settings) { 165 propagateProxySystemProperties(settings); 166 if (requiresProxyAuthentication(settings)) { 167 registerProxyCredentials(settings); 168 } 169 } 170 171 private void initUserAgent(@Nullable String sonarVersion) { 172 userAgent = (sonarVersion == null ? "SonarQube" : String.format("SonarQube %s", sonarVersion)); 173 System.setProperty("http.agent", userAgent); 174 } 175 176 private String getProxySynthesis(URI uri) { 177 return getProxySynthesis(uri, ProxySelector.getDefault()); 178 } 179 180 @VisibleForTesting 181 static String getProxySynthesis(URI uri, ProxySelector proxySelector) { 182 List<Proxy> proxies = proxySelector.select(uri); 183 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) { 184 return "no proxy"; 185 } 186 187 List<String> descriptions = Lists.newArrayList(); 188 for (Proxy proxy : proxies) { 189 if (proxy.type() != Proxy.Type.DIRECT) { 190 descriptions.add(proxy.type() + " proxy: " + proxy.address()); 191 } 192 } 193 194 return Joiner.on(", ").join(descriptions); 195 } 196 197 private void registerProxyCredentials(Map<String, String> settings) { 198 Authenticator.setDefault(new ProxyAuthenticator( 199 settings.get(HTTP_PROXY_USER), 200 settings.get(HTTP_PROXY_PASSWORD))); 201 } 202 203 private boolean requiresProxyAuthentication(Map<String, String> settings) { 204 return settings.containsKey(HTTP_PROXY_USER); 205 } 206 207 private void propagateProxySystemProperties(Map<String, String> settings) { 208 for (String key : PROXY_SETTINGS) { 209 if (settings.containsKey(key)) { 210 System.setProperty(key, settings.get(key)); 211 } 212 } 213 } 214 215 public InputSupplier<InputStream> newInputSupplier(URI uri) { 216 return new HttpInputSupplier(uri, GET, userAgent, null, null, TIMEOUT_MILLISECONDS); 217 } 218 219 public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) { 220 return newInputSupplier(uri, GET, readTimeoutMillis); 221 } 222 223 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, @Nullable Integer readTimeoutMillis) { 224 if (readTimeoutMillis != null) { 225 return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, readTimeoutMillis); 226 } 227 return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, TIMEOUT_MILLISECONDS); 228 } 229 230 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) { 231 return newInputSupplier(uri, GET, login, password); 232 } 233 234 /** 235 * @since 5.0 236 */ 237 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password) { 238 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS); 239 } 240 241 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) { 242 return newInputSupplier(uri, GET, login, password, readTimeoutMillis); 243 } 244 245 /** 246 * @since 5.0 247 */ 248 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password, @Nullable Integer readTimeoutMillis) { 249 if (readTimeoutMillis != null) { 250 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, readTimeoutMillis); 251 } 252 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS); 253 } 254 255 private static class HttpInputSupplier implements InputSupplier<InputStream> { 256 private final String login; 257 private final String password; 258 private final URI uri; 259 private final String userAgent; 260 private final int readTimeoutMillis; 261 private final String requestMethod; 262 263 HttpInputSupplier(URI uri, String requestMethod, String userAgent, String login, String password, int readTimeoutMillis) { 264 this.uri = uri; 265 this.requestMethod = requestMethod; 266 this.userAgent = userAgent; 267 this.login = login; 268 this.password = password; 269 this.readTimeoutMillis = readTimeoutMillis; 270 } 271 272 @Override 273 public InputStream getInput() throws IOException { 274 Loggers.get(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")"); 275 276 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(); 277 connection.setRequestMethod(requestMethod); 278 HttpsTrust.INSTANCE.trust(connection); 279 280 // allow both GZip and Deflate (ZLib) encodings 281 connection.setRequestProperty("Accept-Encoding", "gzip"); 282 if (!Strings.isNullOrEmpty(login)) { 283 String encoded = Base64.encodeBase64String((login + ":" + password).getBytes(Charsets.UTF_8)); 284 connection.setRequestProperty("Authorization", "Basic " + encoded); 285 } 286 connection.setConnectTimeout(TIMEOUT_MILLISECONDS); 287 connection.setReadTimeout(readTimeoutMillis); 288 connection.setUseCaches(true); 289 connection.setInstanceFollowRedirects(true); 290 connection.setRequestProperty("User-Agent", userAgent); 291 292 // establish connection, get response headers 293 connection.connect(); 294 295 // obtain the encoding returned by the server 296 String encoding = connection.getContentEncoding(); 297 298 int responseCode = connection.getResponseCode(); 299 if (responseCode >= 400) { 300 InputStream errorResponse = null; 301 try { 302 errorResponse = connection.getErrorStream(); 303 if (errorResponse != null) { 304 String errorResponseContent = IOUtils.toString(errorResponse); 305 throw new HttpException(uri, responseCode, errorResponseContent); 306 } 307 throw new HttpException(uri, responseCode); 308 309 } finally { 310 IOUtils.closeQuietly(errorResponse); 311 } 312 } 313 314 InputStream resultingInputStream; 315 // create the appropriate stream wrapper based on the encoding type 316 if (encoding != null && "gzip".equalsIgnoreCase(encoding)) { 317 resultingInputStream = new GZIPInputStream(connection.getInputStream()); 318 } else { 319 resultingInputStream = connection.getInputStream(); 320 } 321 return resultingInputStream; 322 } 323 } 324 325 private static class ProxyAuthenticator extends Authenticator { 326 private final PasswordAuthentication auth; 327 328 ProxyAuthenticator(String user, String password) { 329 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray()); 330 } 331 332 @Override 333 protected PasswordAuthentication getPasswordAuthentication() { 334 return auth; 335 } 336 } 337 } 338 339 public static class HttpException extends RuntimeException { 340 private final URI uri; 341 private final int responseCode; 342 private final String responseContent; 343 344 public HttpException(URI uri, int responseContent) { 345 this(uri, responseContent, ""); 346 } 347 348 public HttpException(URI uri, int responseCode, String responseContent) { 349 super("Fail to download [" + uri + "]. Response code: " + responseCode); 350 this.uri = uri; 351 this.responseCode = responseCode; 352 this.responseContent = responseContent; 353 } 354 355 public int getResponseCode() { 356 return responseCode; 357 } 358 359 public URI getUri() { 360 return uri; 361 } 362 363 public String getResponseContent() { 364 return responseContent; 365 } 366 } 367}