001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2013 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.api.utils;
021
022 import com.google.common.annotations.VisibleForTesting;
023 import com.google.common.base.Joiner;
024 import com.google.common.base.Strings;
025 import com.google.common.collect.ImmutableList;
026 import com.google.common.collect.Lists;
027 import com.google.common.io.ByteStreams;
028 import com.google.common.io.CharStreams;
029 import com.google.common.io.Files;
030 import com.google.common.io.InputSupplier;
031 import org.apache.commons.codec.binary.Base64;
032 import org.apache.commons.io.FileUtils;
033 import org.apache.commons.io.IOUtils;
034 import org.slf4j.LoggerFactory;
035 import org.sonar.api.BatchComponent;
036 import org.sonar.api.ServerComponent;
037 import org.sonar.api.config.Settings;
038 import org.sonar.api.platform.Server;
039
040 import javax.annotation.Nullable;
041
042 import java.io.File;
043 import java.io.IOException;
044 import java.io.InputStream;
045 import java.net.Authenticator;
046 import java.net.HttpURLConnection;
047 import java.net.PasswordAuthentication;
048 import java.net.Proxy;
049 import java.net.ProxySelector;
050 import java.net.URI;
051 import java.nio.charset.Charset;
052 import java.util.List;
053 import java.util.Map;
054
055 /**
056 * This component downloads HTTP files
057 *
058 * @since 2.2
059 */
060 public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
061 public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
062
063 private final BaseHttpDownloader downloader;
064 private final Integer readTimeout;
065
066 public HttpDownloader(Server server, Settings settings) {
067 this(server, settings, null);
068 }
069
070 public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) {
071 this.readTimeout = readTimeout;
072 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
073 }
074
075 public HttpDownloader(Settings settings) {
076 this(settings, null);
077 }
078
079 public HttpDownloader(Settings settings, @Nullable Integer readTimeout) {
080 this.readTimeout = readTimeout;
081 downloader = new BaseHttpDownloader(settings.getProperties(), null);
082 }
083
084 @Override
085 String description(URI uri) {
086 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
087 }
088
089 @Override
090 String[] getSupportedSchemes() {
091 return new String[] {"http", "https"};
092 }
093
094 @Override
095 byte[] readBytes(URI uri) {
096 return download(uri);
097 }
098
099 @Override
100 String readString(URI uri, Charset charset) {
101 try {
102 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset));
103 } catch (IOException e) {
104 throw failToDownload(uri, e);
105 }
106 }
107
108 public String downloadPlainText(URI uri, String encoding) {
109 return readString(uri, Charset.forName(encoding));
110 }
111
112 public byte[] download(URI uri) {
113 try {
114 return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout));
115 } catch (IOException e) {
116 throw failToDownload(uri, e);
117 }
118 }
119
120 public String getProxySynthesis(URI uri) {
121 return downloader.getProxySynthesis(uri);
122 }
123
124 public InputStream openStream(URI uri) {
125 try {
126 return downloader.newInputSupplier(uri, this.readTimeout).getInput();
127 } catch (IOException e) {
128 throw failToDownload(uri, e);
129 }
130 }
131
132 public void download(URI uri, File toFile) {
133 try {
134 Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile);
135 } catch (IOException e) {
136 FileUtils.deleteQuietly(toFile);
137 throw failToDownload(uri, e);
138 }
139 }
140
141 private SonarException failToDownload(URI uri, IOException e) {
142 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
143 }
144
145 public static class BaseHttpDownloader {
146 private static final List<String> PROXY_SETTINGS = ImmutableList.of(
147 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
148 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
149
150 private String userAgent;
151
152 public BaseHttpDownloader(Map<String, String> settings, String userAgent) {
153 initProxy(settings);
154 initUserAgent(userAgent);
155 }
156
157 private void initProxy(Map<String, String> settings) {
158 propagateProxySystemProperties(settings);
159 if (requiresProxyAuthentication(settings)) {
160 registerProxyCredentials(settings);
161 }
162 }
163
164 private void initUserAgent(String sonarVersion) {
165 userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion));
166 System.setProperty("http.agent", userAgent);
167 }
168
169 private String getProxySynthesis(URI uri) {
170 return getProxySynthesis(uri, ProxySelector.getDefault());
171 }
172
173 @VisibleForTesting
174 static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
175 List<Proxy> proxies = proxySelector.select(uri);
176 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
177 return "no proxy";
178 }
179
180 List<String> descriptions = Lists.newArrayList();
181 for (Proxy proxy : proxies) {
182 if (proxy.type() != Proxy.Type.DIRECT) {
183 descriptions.add("proxy: " + proxy.address());
184 }
185 }
186
187 return Joiner.on(", ").join(descriptions);
188 }
189
190 private void registerProxyCredentials(Map<String, String> settings) {
191 Authenticator.setDefault(new ProxyAuthenticator(
192 settings.get("http.proxyUser"),
193 settings.get("http.proxyPassword")));
194 }
195
196 private boolean requiresProxyAuthentication(Map<String, String> settings) {
197 return settings.containsKey("http.proxyUser");
198 }
199
200 private void propagateProxySystemProperties(Map<String, String> settings) {
201 for (String key : PROXY_SETTINGS) {
202 if (settings.containsKey(key)) {
203 System.setProperty(key, settings.get(key));
204 }
205 }
206 }
207
208 public InputSupplier<InputStream> newInputSupplier(URI uri) {
209 return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
210 }
211
212 public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) {
213 if (readTimeoutMillis != null) {
214 return new HttpInputSupplier(uri, userAgent, null, null, readTimeoutMillis);
215 }
216 return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
217 }
218
219 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
220 return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
221 }
222
223 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) {
224 if (readTimeoutMillis != null) {
225 return new HttpInputSupplier(uri, userAgent, login, password, readTimeoutMillis);
226 }
227 return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
228 }
229
230 private static class HttpInputSupplier implements InputSupplier<InputStream> {
231 private final String login;
232 private final String password;
233 private final URI uri;
234 private final String userAgent;
235 private final int readTimeoutMillis;
236
237 HttpInputSupplier(URI uri, String userAgent, String login, String password, int readTimeoutMillis) {
238 this.uri = uri;
239 this.userAgent = userAgent;
240 this.login = login;
241 this.password = password;
242 this.readTimeoutMillis = readTimeoutMillis;
243 }
244
245 public InputStream getInput() throws IOException {
246 LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
247
248 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
249 if (!Strings.isNullOrEmpty(login)) {
250 String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes()));
251 connection.setRequestProperty("Authorization", "Basic " + encoded);
252 }
253 connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
254 connection.setReadTimeout(readTimeoutMillis);
255 connection.setUseCaches(true);
256 connection.setInstanceFollowRedirects(true);
257 connection.setRequestProperty("User-Agent", userAgent);
258
259 int responseCode = connection.getResponseCode();
260 if (responseCode >= 400) {
261 InputStream errorResponse = null;
262 try {
263 errorResponse = connection.getErrorStream();
264 if (errorResponse != null) {
265 String errorResponseContent = IOUtils.toString(errorResponse);
266 throw new HttpException(uri, responseCode, errorResponseContent);
267 }
268 else {
269 throw new HttpException(uri, responseCode);
270 }
271 } finally {
272 IOUtils.closeQuietly(errorResponse);
273 }
274 }
275
276 return connection.getInputStream();
277 }
278 }
279
280 private static class ProxyAuthenticator extends Authenticator {
281 private final PasswordAuthentication auth;
282
283 ProxyAuthenticator(String user, String password) {
284 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
285 }
286
287 @Override
288 protected PasswordAuthentication getPasswordAuthentication() {
289 return auth;
290 }
291 }
292 }
293
294 public static class HttpException extends RuntimeException {
295 private final URI uri;
296 private final int responseCode;
297 private final String responseContent;
298
299 public HttpException(URI uri, int responseContent) {
300 this(uri, responseContent, null);
301 }
302
303 public HttpException(URI uri, int responseCode, String responseContent) {
304 super("Fail to download [" + uri + "]. Response code: " + responseCode);
305 this.uri = uri;
306 this.responseCode = responseCode;
307 this.responseContent = responseContent;
308 }
309
310 public int getResponseCode() {
311 return responseCode;
312 }
313
314 public URI getUri() {
315 return uri;
316 }
317
318 public String getResponseContent() {
319 return responseContent;
320 }
321 }
322 }