001 /* 002 * SonarQube, open source software quality management tool. 003 * Copyright (C) 2008-2014 SonarSource 004 * mailto:contact AT sonarsource DOT com 005 * 006 * SonarQube is free software; you can redistribute it and/or 007 * modify it under the terms of the GNU Lesser General Public 008 * License as published by the Free Software Foundation; either 009 * version 3 of the License, or (at your option) any later version. 010 * 011 * SonarQube is distributed in the hope that it will be useful, 012 * but WITHOUT ANY WARRANTY; without even the implied warranty of 013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 014 * Lesser General Public License for more details. 015 * 016 * You should have received a copy of the GNU Lesser General Public License 017 * along with this program; if not, write to the Free Software Foundation, 018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 019 */ 020 package org.sonar.api.utils; 021 022 import com.google.common.annotations.VisibleForTesting; 023 import com.google.common.base.Joiner; 024 import com.google.common.base.Strings; 025 import com.google.common.collect.ImmutableList; 026 import com.google.common.collect.Lists; 027 import com.google.common.io.ByteStreams; 028 import com.google.common.io.CharStreams; 029 import com.google.common.io.Files; 030 import com.google.common.io.InputSupplier; 031 import org.apache.commons.codec.binary.Base64; 032 import org.apache.commons.io.FileUtils; 033 import org.apache.commons.io.IOUtils; 034 import org.slf4j.LoggerFactory; 035 import org.sonar.api.BatchComponent; 036 import org.sonar.api.ServerComponent; 037 import org.sonar.api.config.Settings; 038 import org.sonar.api.platform.Server; 039 040 import javax.annotation.Nullable; 041 042 import java.io.File; 043 import java.io.IOException; 044 import java.io.InputStream; 045 import java.net.Authenticator; 046 import java.net.HttpURLConnection; 047 import java.net.PasswordAuthentication; 048 import java.net.Proxy; 049 import java.net.ProxySelector; 050 import java.net.URI; 051 import java.nio.charset.Charset; 052 import java.util.List; 053 import java.util.Map; 054 import java.util.zip.GZIPInputStream; 055 056 /** 057 * This component downloads HTTP files 058 * 059 * @since 2.2 060 */ 061 public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent { 062 public static final int TIMEOUT_MILLISECONDS = 20 * 1000; 063 064 private final BaseHttpDownloader downloader; 065 private final Integer readTimeout; 066 067 public HttpDownloader(Server server, Settings settings) { 068 this(server, settings, null); 069 } 070 071 public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) { 072 this.readTimeout = readTimeout; 073 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion()); 074 } 075 076 public HttpDownloader(Settings settings) { 077 this(settings, null); 078 } 079 080 public HttpDownloader(Settings settings, @Nullable Integer readTimeout) { 081 this.readTimeout = readTimeout; 082 downloader = new BaseHttpDownloader(settings.getProperties(), null); 083 } 084 085 @Override 086 String description(URI uri) { 087 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri)); 088 } 089 090 @Override 091 String[] getSupportedSchemes() { 092 return new String[] {"http", "https"}; 093 } 094 095 @Override 096 byte[] readBytes(URI uri) { 097 return download(uri); 098 } 099 100 @Override 101 String readString(URI uri, Charset charset) { 102 try { 103 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset)); 104 } catch (IOException e) { 105 throw failToDownload(uri, e); 106 } 107 } 108 109 public String downloadPlainText(URI uri, String encoding) { 110 return readString(uri, Charset.forName(encoding)); 111 } 112 113 public byte[] download(URI uri) { 114 try { 115 return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout)); 116 } catch (IOException e) { 117 throw failToDownload(uri, e); 118 } 119 } 120 121 public String getProxySynthesis(URI uri) { 122 return downloader.getProxySynthesis(uri); 123 } 124 125 public InputStream openStream(URI uri) { 126 try { 127 return downloader.newInputSupplier(uri, this.readTimeout).getInput(); 128 } catch (IOException e) { 129 throw failToDownload(uri, e); 130 } 131 } 132 133 public void download(URI uri, File toFile) { 134 try { 135 Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile); 136 } catch (IOException e) { 137 FileUtils.deleteQuietly(toFile); 138 throw failToDownload(uri, e); 139 } 140 } 141 142 private SonarException failToDownload(URI uri, IOException e) { 143 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e); 144 } 145 146 public static class BaseHttpDownloader { 147 148 private static final String HTTP_PROXY_USER = "http.proxyUser"; 149 private static final String HTTP_PROXY_PASSWORD = "http.proxyPassword"; 150 151 private static final List<String> PROXY_SETTINGS = ImmutableList.of( 152 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts", 153 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort"); 154 155 private String userAgent; 156 157 public BaseHttpDownloader(Map<String, String> settings, String userAgent) { 158 initProxy(settings); 159 initUserAgent(userAgent); 160 } 161 162 private void initProxy(Map<String, String> settings) { 163 propagateProxySystemProperties(settings); 164 if (requiresProxyAuthentication(settings)) { 165 registerProxyCredentials(settings); 166 } 167 } 168 169 private void initUserAgent(String sonarVersion) { 170 userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion)); 171 System.setProperty("http.agent", userAgent); 172 } 173 174 private String getProxySynthesis(URI uri) { 175 return getProxySynthesis(uri, ProxySelector.getDefault()); 176 } 177 178 @VisibleForTesting 179 static String getProxySynthesis(URI uri, ProxySelector proxySelector) { 180 List<Proxy> proxies = proxySelector.select(uri); 181 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) { 182 return "no proxy"; 183 } 184 185 List<String> descriptions = Lists.newArrayList(); 186 for (Proxy proxy : proxies) { 187 if (proxy.type() != Proxy.Type.DIRECT) { 188 descriptions.add(proxy.type() + " proxy: " + proxy.address()); 189 } 190 } 191 192 return Joiner.on(", ").join(descriptions); 193 } 194 195 private void registerProxyCredentials(Map<String, String> settings) { 196 Authenticator.setDefault(new ProxyAuthenticator( 197 settings.get(HTTP_PROXY_USER), 198 settings.get(HTTP_PROXY_PASSWORD))); 199 } 200 201 private boolean requiresProxyAuthentication(Map<String, String> settings) { 202 return settings.containsKey(HTTP_PROXY_USER); 203 } 204 205 private void propagateProxySystemProperties(Map<String, String> settings) { 206 for (String key : PROXY_SETTINGS) { 207 if (settings.containsKey(key)) { 208 System.setProperty(key, settings.get(key)); 209 } 210 } 211 } 212 213 public InputSupplier<InputStream> newInputSupplier(URI uri) { 214 return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS); 215 } 216 217 public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) { 218 if (readTimeoutMillis != null) { 219 return new HttpInputSupplier(uri, userAgent, null, null, readTimeoutMillis); 220 } 221 return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS); 222 } 223 224 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) { 225 return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS); 226 } 227 228 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) { 229 if (readTimeoutMillis != null) { 230 return new HttpInputSupplier(uri, userAgent, login, password, readTimeoutMillis); 231 } 232 return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS); 233 } 234 235 private static class HttpInputSupplier implements InputSupplier<InputStream> { 236 private final String login; 237 private final String password; 238 private final URI uri; 239 private final String userAgent; 240 private final int readTimeoutMillis; 241 242 HttpInputSupplier(URI uri, String userAgent, String login, String password, int readTimeoutMillis) { 243 this.uri = uri; 244 this.userAgent = userAgent; 245 this.login = login; 246 this.password = password; 247 this.readTimeoutMillis = readTimeoutMillis; 248 } 249 250 public InputStream getInput() throws IOException { 251 LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")"); 252 253 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(); 254 HttpsTrust.INSTANCE.trust(connection); 255 256 // allow both GZip and Deflate (ZLib) encodings 257 connection.setRequestProperty("Accept-Encoding", "gzip"); 258 if (!Strings.isNullOrEmpty(login)) { 259 String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes())); 260 connection.setRequestProperty("Authorization", "Basic " + encoded); 261 } 262 connection.setConnectTimeout(TIMEOUT_MILLISECONDS); 263 connection.setReadTimeout(readTimeoutMillis); 264 connection.setUseCaches(true); 265 connection.setInstanceFollowRedirects(true); 266 connection.setRequestProperty("User-Agent", userAgent); 267 268 // establish connection, get response headers 269 connection.connect(); 270 271 // obtain the encoding returned by the server 272 String encoding = connection.getContentEncoding(); 273 274 int responseCode = connection.getResponseCode(); 275 if (responseCode >= 400) { 276 InputStream errorResponse = null; 277 try { 278 errorResponse = connection.getErrorStream(); 279 if (errorResponse != null) { 280 String errorResponseContent = IOUtils.toString(errorResponse); 281 throw new HttpException(uri, responseCode, errorResponseContent); 282 } 283 throw new HttpException(uri, responseCode); 284 285 } finally { 286 IOUtils.closeQuietly(errorResponse); 287 } 288 } 289 290 InputStream resultingInputStream; 291 // create the appropriate stream wrapper based on the encoding type 292 if (encoding != null && "gzip".equalsIgnoreCase(encoding)) { 293 resultingInputStream = new GZIPInputStream(connection.getInputStream()); 294 } else { 295 resultingInputStream = connection.getInputStream(); 296 } 297 return resultingInputStream; 298 } 299 } 300 301 private static class ProxyAuthenticator extends Authenticator { 302 private final PasswordAuthentication auth; 303 304 ProxyAuthenticator(String user, String password) { 305 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray()); 306 } 307 308 @Override 309 protected PasswordAuthentication getPasswordAuthentication() { 310 return auth; 311 } 312 } 313 } 314 315 public static class HttpException extends RuntimeException { 316 private final URI uri; 317 private final int responseCode; 318 private final String responseContent; 319 320 public HttpException(URI uri, int responseContent) { 321 this(uri, responseContent, null); 322 } 323 324 public HttpException(URI uri, int responseCode, String responseContent) { 325 super("Fail to download [" + uri + "]. Response code: " + responseCode); 326 this.uri = uri; 327 this.responseCode = responseCode; 328 this.responseContent = responseContent; 329 } 330 331 public int getResponseCode() { 332 return responseCode; 333 } 334 335 public URI getUri() { 336 return uri; 337 } 338 339 public String getResponseContent() { 340 return responseContent; 341 } 342 } 343 }