001 /* 002 * SonarQube, open source software quality management tool. 003 * Copyright (C) 2008-2014 SonarSource 004 * mailto:contact AT sonarsource DOT com 005 * 006 * SonarQube is free software; you can redistribute it and/or 007 * modify it under the terms of the GNU Lesser General Public 008 * License as published by the Free Software Foundation; either 009 * version 3 of the License, or (at your option) any later version. 010 * 011 * SonarQube is distributed in the hope that it will be useful, 012 * but WITHOUT ANY WARRANTY; without even the implied warranty of 013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 014 * Lesser General Public License for more details. 015 * 016 * You should have received a copy of the GNU Lesser General Public License 017 * along with this program; if not, write to the Free Software Foundation, 018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 019 */ 020 package org.sonar.api.utils; 021 022 import com.google.common.annotations.VisibleForTesting; 023 import com.google.common.base.Joiner; 024 import com.google.common.base.Strings; 025 import com.google.common.collect.ImmutableList; 026 import com.google.common.collect.Lists; 027 import com.google.common.io.ByteStreams; 028 import com.google.common.io.CharStreams; 029 import com.google.common.io.Files; 030 import com.google.common.io.InputSupplier; 031 import org.apache.commons.codec.binary.Base64; 032 import org.apache.commons.io.FileUtils; 033 import org.apache.commons.io.IOUtils; 034 import org.slf4j.LoggerFactory; 035 import org.sonar.api.BatchComponent; 036 import org.sonar.api.ServerComponent; 037 import org.sonar.api.config.Settings; 038 import org.sonar.api.platform.Server; 039 040 import javax.annotation.Nullable; 041 042 import java.io.File; 043 import java.io.IOException; 044 import java.io.InputStream; 045 import java.net.Authenticator; 046 import java.net.HttpURLConnection; 047 import java.net.PasswordAuthentication; 048 import java.net.Proxy; 049 import java.net.ProxySelector; 050 import java.net.URI; 051 import java.nio.charset.Charset; 052 import java.util.List; 053 import java.util.Map; 054 import java.util.zip.GZIPInputStream; 055 056 /** 057 * This component downloads HTTP files 058 * 059 * @since 2.2 060 */ 061 public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent { 062 public static final int TIMEOUT_MILLISECONDS = 20 * 1000; 063 064 private final BaseHttpDownloader downloader; 065 private final Integer readTimeout; 066 067 public HttpDownloader(Server server, Settings settings) { 068 this(server, settings, null); 069 } 070 071 public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) { 072 this.readTimeout = readTimeout; 073 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion()); 074 } 075 076 public HttpDownloader(Settings settings) { 077 this(settings, null); 078 } 079 080 public HttpDownloader(Settings settings, @Nullable Integer readTimeout) { 081 this.readTimeout = readTimeout; 082 downloader = new BaseHttpDownloader(settings.getProperties(), null); 083 } 084 085 @Override 086 String description(URI uri) { 087 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri)); 088 } 089 090 @Override 091 String[] getSupportedSchemes() { 092 return new String[] {"http", "https"}; 093 } 094 095 @Override 096 byte[] readBytes(URI uri) { 097 return download(uri); 098 } 099 100 @Override 101 String readString(URI uri, Charset charset) { 102 try { 103 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset)); 104 } catch (IOException e) { 105 throw failToDownload(uri, e); 106 } 107 } 108 109 public String downloadPlainText(URI uri, String encoding) { 110 return readString(uri, Charset.forName(encoding)); 111 } 112 113 public byte[] download(URI uri) { 114 try { 115 return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout)); 116 } catch (IOException e) { 117 throw failToDownload(uri, e); 118 } 119 } 120 121 public String getProxySynthesis(URI uri) { 122 return downloader.getProxySynthesis(uri); 123 } 124 125 public InputStream openStream(URI uri) { 126 try { 127 return downloader.newInputSupplier(uri, this.readTimeout).getInput(); 128 } catch (IOException e) { 129 throw failToDownload(uri, e); 130 } 131 } 132 133 public void download(URI uri, File toFile) { 134 try { 135 Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile); 136 } catch (IOException e) { 137 FileUtils.deleteQuietly(toFile); 138 throw failToDownload(uri, e); 139 } 140 } 141 142 private SonarException failToDownload(URI uri, IOException e) { 143 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e); 144 } 145 146 public static class BaseHttpDownloader { 147 148 private static final String GET = "GET"; 149 private static final String HTTP_PROXY_USER = "http.proxyUser"; 150 private static final String HTTP_PROXY_PASSWORD = "http.proxyPassword"; 151 152 private static final List<String> PROXY_SETTINGS = ImmutableList.of( 153 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts", 154 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort"); 155 156 private String userAgent; 157 158 public BaseHttpDownloader(Map<String, String> settings, @Nullable String userAgent) { 159 initProxy(settings); 160 initUserAgent(userAgent); 161 } 162 163 private void initProxy(Map<String, String> settings) { 164 propagateProxySystemProperties(settings); 165 if (requiresProxyAuthentication(settings)) { 166 registerProxyCredentials(settings); 167 } 168 } 169 170 private void initUserAgent(@Nullable String sonarVersion) { 171 userAgent = (sonarVersion == null ? "SonarQube" : String.format("SonarQube %s", sonarVersion)); 172 System.setProperty("http.agent", userAgent); 173 } 174 175 private String getProxySynthesis(URI uri) { 176 return getProxySynthesis(uri, ProxySelector.getDefault()); 177 } 178 179 @VisibleForTesting 180 static String getProxySynthesis(URI uri, ProxySelector proxySelector) { 181 List<Proxy> proxies = proxySelector.select(uri); 182 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) { 183 return "no proxy"; 184 } 185 186 List<String> descriptions = Lists.newArrayList(); 187 for (Proxy proxy : proxies) { 188 if (proxy.type() != Proxy.Type.DIRECT) { 189 descriptions.add(proxy.type() + " proxy: " + proxy.address()); 190 } 191 } 192 193 return Joiner.on(", ").join(descriptions); 194 } 195 196 private void registerProxyCredentials(Map<String, String> settings) { 197 Authenticator.setDefault(new ProxyAuthenticator( 198 settings.get(HTTP_PROXY_USER), 199 settings.get(HTTP_PROXY_PASSWORD))); 200 } 201 202 private boolean requiresProxyAuthentication(Map<String, String> settings) { 203 return settings.containsKey(HTTP_PROXY_USER); 204 } 205 206 private void propagateProxySystemProperties(Map<String, String> settings) { 207 for (String key : PROXY_SETTINGS) { 208 if (settings.containsKey(key)) { 209 System.setProperty(key, settings.get(key)); 210 } 211 } 212 } 213 214 public InputSupplier<InputStream> newInputSupplier(URI uri) { 215 return new HttpInputSupplier(uri, GET, userAgent, null, null, TIMEOUT_MILLISECONDS); 216 } 217 218 public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) { 219 return newInputSupplier(uri, GET, readTimeoutMillis); 220 } 221 222 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, @Nullable Integer readTimeoutMillis) { 223 if (readTimeoutMillis != null) { 224 return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, readTimeoutMillis); 225 } 226 return new HttpInputSupplier(uri, requestMethod, userAgent, null, null, TIMEOUT_MILLISECONDS); 227 } 228 229 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) { 230 return newInputSupplier(uri, GET, login, password); 231 } 232 233 /** 234 * @since 5.0 235 */ 236 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password) { 237 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS); 238 } 239 240 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) { 241 return newInputSupplier(uri, GET, login, password, readTimeoutMillis); 242 } 243 244 /** 245 * @since 5.0 246 */ 247 public InputSupplier<InputStream> newInputSupplier(URI uri, String requestMethod, String login, String password, @Nullable Integer readTimeoutMillis) { 248 if (readTimeoutMillis != null) { 249 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, readTimeoutMillis); 250 } 251 return new HttpInputSupplier(uri, requestMethod, userAgent, login, password, TIMEOUT_MILLISECONDS); 252 } 253 254 private static class HttpInputSupplier implements InputSupplier<InputStream> { 255 private final String login; 256 private final String password; 257 private final URI uri; 258 private final String userAgent; 259 private final int readTimeoutMillis; 260 private final String requestMethod; 261 262 HttpInputSupplier(URI uri, String requestMethod, String userAgent, String login, String password, int readTimeoutMillis) { 263 this.uri = uri; 264 this.requestMethod = requestMethod; 265 this.userAgent = userAgent; 266 this.login = login; 267 this.password = password; 268 this.readTimeoutMillis = readTimeoutMillis; 269 } 270 271 @Override 272 public InputStream getInput() throws IOException { 273 LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")"); 274 275 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(); 276 connection.setRequestMethod(requestMethod); 277 HttpsTrust.INSTANCE.trust(connection); 278 279 // allow both GZip and Deflate (ZLib) encodings 280 connection.setRequestProperty("Accept-Encoding", "gzip"); 281 if (!Strings.isNullOrEmpty(login)) { 282 String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes())); 283 connection.setRequestProperty("Authorization", "Basic " + encoded); 284 } 285 connection.setConnectTimeout(TIMEOUT_MILLISECONDS); 286 connection.setReadTimeout(readTimeoutMillis); 287 connection.setUseCaches(true); 288 connection.setInstanceFollowRedirects(true); 289 connection.setRequestProperty("User-Agent", userAgent); 290 291 // establish connection, get response headers 292 connection.connect(); 293 294 // obtain the encoding returned by the server 295 String encoding = connection.getContentEncoding(); 296 297 int responseCode = connection.getResponseCode(); 298 if (responseCode >= 400) { 299 InputStream errorResponse = null; 300 try { 301 errorResponse = connection.getErrorStream(); 302 if (errorResponse != null) { 303 String errorResponseContent = IOUtils.toString(errorResponse); 304 throw new HttpException(uri, responseCode, errorResponseContent); 305 } 306 throw new HttpException(uri, responseCode); 307 308 } finally { 309 IOUtils.closeQuietly(errorResponse); 310 } 311 } 312 313 InputStream resultingInputStream; 314 // create the appropriate stream wrapper based on the encoding type 315 if (encoding != null && "gzip".equalsIgnoreCase(encoding)) { 316 resultingInputStream = new GZIPInputStream(connection.getInputStream()); 317 } else { 318 resultingInputStream = connection.getInputStream(); 319 } 320 return resultingInputStream; 321 } 322 } 323 324 private static class ProxyAuthenticator extends Authenticator { 325 private final PasswordAuthentication auth; 326 327 ProxyAuthenticator(String user, String password) { 328 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray()); 329 } 330 331 @Override 332 protected PasswordAuthentication getPasswordAuthentication() { 333 return auth; 334 } 335 } 336 } 337 338 public static class HttpException extends RuntimeException { 339 private final URI uri; 340 private final int responseCode; 341 private final String responseContent; 342 343 public HttpException(URI uri, int responseContent) { 344 this(uri, responseContent, ""); 345 } 346 347 public HttpException(URI uri, int responseCode, String responseContent) { 348 super("Fail to download [" + uri + "]. Response code: " + responseCode); 349 this.uri = uri; 350 this.responseCode = responseCode; 351 this.responseContent = responseContent; 352 } 353 354 public int getResponseCode() { 355 return responseCode; 356 } 357 358 public URI getUri() { 359 return uri; 360 } 361 362 public String getResponseContent() { 363 return responseContent; 364 } 365 } 366 }