001/* 002 * Sonar, open source software quality management tool. 003 * Copyright (C) 2008-2012 SonarSource 004 * mailto:contact AT sonarsource DOT com 005 * 006 * Sonar is free software; you can redistribute it and/or 007 * modify it under the terms of the GNU Lesser General Public 008 * License as published by the Free Software Foundation; either 009 * version 3 of the License, or (at your option) any later version. 010 * 011 * Sonar is distributed in the hope that it will be useful, 012 * but WITHOUT ANY WARRANTY; without even the implied warranty of 013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 014 * Lesser General Public License for more details. 015 * 016 * You should have received a copy of the GNU Lesser General Public 017 * License along with Sonar; if not, write to the Free Software 018 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 019 */ 020package org.sonar.api.utils; 021 022import com.google.common.annotations.VisibleForTesting; 023import com.google.common.base.Joiner; 024import com.google.common.base.Strings; 025import com.google.common.collect.ImmutableList; 026import com.google.common.collect.Lists; 027import com.google.common.io.ByteStreams; 028import com.google.common.io.CharStreams; 029import com.google.common.io.Files; 030import com.google.common.io.InputSupplier; 031import org.apache.commons.codec.binary.Base64; 032import org.apache.commons.io.FileUtils; 033import org.slf4j.LoggerFactory; 034import org.sonar.api.BatchComponent; 035import org.sonar.api.ServerComponent; 036import org.sonar.api.config.Settings; 037import org.sonar.api.platform.Server; 038 039import java.io.File; 040import java.io.IOException; 041import java.io.InputStream; 042import java.net.Authenticator; 043import java.net.HttpURLConnection; 044import java.net.PasswordAuthentication; 045import java.net.Proxy; 046import java.net.ProxySelector; 047import java.net.URI; 048import java.nio.charset.Charset; 049import java.util.List; 050import java.util.Map; 051 052/** 053 * This component downloads HTTP files 054 * 055 * @since 2.2 056 */ 057public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent { 058 public static final int TIMEOUT_MILLISECONDS = 20 * 1000; 059 060 private final BaseHttpDownloader downloader; 061 062 public HttpDownloader(Server server, Settings settings) { 063 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion()); 064 } 065 066 public HttpDownloader(Settings settings) { 067 downloader = new BaseHttpDownloader(settings.getProperties(), null); 068 } 069 070 @Override 071 String description(URI uri) { 072 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri)); 073 } 074 075 @Override 076 String[] getSupportedSchemes() { 077 return new String[]{"http", "https"}; 078 } 079 080 @Override 081 byte[] readBytes(URI uri) { 082 return download(uri); 083 } 084 085 @Override 086 String readString(URI uri, Charset charset) { 087 try { 088 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri), charset)); 089 } catch (IOException e) { 090 throw failToDownload(uri, e); 091 } 092 } 093 094 public String downloadPlainText(URI uri, String encoding) { 095 return readString(uri, Charset.forName(encoding)); 096 } 097 098 public byte[] download(URI uri) { 099 try { 100 return ByteStreams.toByteArray(downloader.newInputSupplier(uri)); 101 } catch (IOException e) { 102 throw failToDownload(uri, e); 103 } 104 } 105 106 public String getProxySynthesis(URI uri) { 107 return downloader.getProxySynthesis(uri); 108 } 109 110 public InputStream openStream(URI uri) { 111 try { 112 return downloader.newInputSupplier(uri).getInput(); 113 } catch (IOException e) { 114 throw failToDownload(uri, e); 115 } 116 } 117 118 public void download(URI uri, File toFile) { 119 try { 120 Files.copy(downloader.newInputSupplier(uri), toFile); 121 } catch (IOException e) { 122 FileUtils.deleteQuietly(toFile); 123 throw failToDownload(uri, e); 124 } 125 } 126 127 private SonarException failToDownload(URI uri, IOException e) { 128 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e); 129 } 130 131 public static class BaseHttpDownloader { 132 private static final List<String> PROXY_SETTINGS = ImmutableList.of( 133 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts", 134 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort"); 135 136 private String userAgent; 137 138 public BaseHttpDownloader(Map<String, String> settings, String userAgent) { 139 initProxy(settings); 140 initUserAgent(userAgent); 141 } 142 143 private void initProxy(Map<String, String> settings) { 144 propagateProxySystemProperties(settings); 145 if (requiresProxyAuthentication(settings)) { 146 registerProxyCredentials(settings); 147 } 148 } 149 150 private void initUserAgent(String sonarVersion) { 151 userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion)); 152 System.setProperty("http.agent", userAgent); 153 } 154 155 private String getProxySynthesis(URI uri) { 156 return getProxySynthesis(uri, ProxySelector.getDefault()); 157 } 158 159 @VisibleForTesting 160 static String getProxySynthesis(URI uri, ProxySelector proxySelector) { 161 List<Proxy> proxies = proxySelector.select(uri); 162 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) { 163 return "no proxy"; 164 } 165 166 List<String> descriptions = Lists.newArrayList(); 167 for (Proxy proxy : proxies) { 168 if (proxy.type() != Proxy.Type.DIRECT) { 169 descriptions.add("proxy: " + proxy.address()); 170 } 171 } 172 173 return Joiner.on(", ").join(descriptions); 174 } 175 176 private void registerProxyCredentials(Map<String, String> settings) { 177 Authenticator.setDefault(new ProxyAuthenticator( 178 settings.get("http.proxyUser"), 179 settings.get("http.proxyPassword"))); 180 } 181 182 private boolean requiresProxyAuthentication(Map<String, String> settings) { 183 return settings.containsKey("http.proxyUser"); 184 } 185 186 private void propagateProxySystemProperties(Map<String, String> settings) { 187 for (String key : PROXY_SETTINGS) { 188 if (settings.containsKey(key)) { 189 System.setProperty(key, settings.get(key)); 190 } 191 } 192 } 193 194 public InputSupplier<InputStream> newInputSupplier(URI uri) { 195 return new HttpInputSupplier(uri, userAgent, null, null); 196 } 197 198 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) { 199 return new HttpInputSupplier(uri, userAgent, login, password); 200 } 201 202 private static class HttpInputSupplier implements InputSupplier<InputStream> { 203 private final String login; 204 private final String password; 205 private final URI uri; 206 private final String userAgent; 207 208 HttpInputSupplier(URI uri, String userAgent, String login, String password) { 209 this.uri = uri; 210 this.userAgent = userAgent; 211 this.login = login; 212 this.password = password; 213 } 214 215 public InputStream getInput() throws IOException { 216 LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")"); 217 218 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(); 219 if (!Strings.isNullOrEmpty(login)) { 220 String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes())); 221 connection.setRequestProperty("Authorization", "Basic " + encoded); 222 } 223 connection.setConnectTimeout(TIMEOUT_MILLISECONDS); 224 connection.setReadTimeout(TIMEOUT_MILLISECONDS); 225 connection.setUseCaches(true); 226 connection.setInstanceFollowRedirects(true); 227 connection.setRequestProperty("User-Agent", userAgent); 228 229 int responseCode = connection.getResponseCode(); 230 if (responseCode >= 400) { 231 throw new HttpException(uri, responseCode); 232 } 233 234 return connection.getInputStream(); 235 } 236 } 237 238 private static class ProxyAuthenticator extends Authenticator { 239 private final PasswordAuthentication auth; 240 241 ProxyAuthenticator(String user, String password) { 242 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray()); 243 } 244 245 @Override 246 protected PasswordAuthentication getPasswordAuthentication() { 247 return auth; 248 } 249 } 250 } 251 252 public static class HttpException extends RuntimeException { 253 private final URI uri; 254 private final int responseCode; 255 256 public HttpException(URI uri, int responseCode) { 257 super("Fail to download [" + uri + "]. Response code: " + responseCode); 258 this.uri = uri; 259 this.responseCode = responseCode; 260 } 261 262 public int getResponseCode() { 263 return responseCode; 264 } 265 266 public URI getUri() { 267 return uri; 268 } 269 } 270}