001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2013 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.api.utils;
021
022 import com.google.common.annotations.VisibleForTesting;
023 import com.google.common.base.Joiner;
024 import com.google.common.base.Strings;
025 import com.google.common.collect.ImmutableList;
026 import com.google.common.collect.Lists;
027 import com.google.common.io.ByteStreams;
028 import com.google.common.io.CharStreams;
029 import com.google.common.io.Files;
030 import com.google.common.io.InputSupplier;
031 import org.apache.commons.codec.binary.Base64;
032 import org.apache.commons.io.FileUtils;
033 import org.slf4j.LoggerFactory;
034 import org.sonar.api.BatchComponent;
035 import org.sonar.api.ServerComponent;
036 import org.sonar.api.config.Settings;
037 import org.sonar.api.platform.Server;
038
039 import java.io.File;
040 import java.io.IOException;
041 import java.io.InputStream;
042 import java.net.Authenticator;
043 import java.net.HttpURLConnection;
044 import java.net.PasswordAuthentication;
045 import java.net.Proxy;
046 import java.net.ProxySelector;
047 import java.net.URI;
048 import java.nio.charset.Charset;
049 import java.util.List;
050 import java.util.Map;
051
052 /**
053 * This component downloads HTTP files
054 *
055 * @since 2.2
056 */
057 public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
058 public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
059
060 private final BaseHttpDownloader downloader;
061
062 public HttpDownloader(Server server, Settings settings) {
063 downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
064 }
065
066 public HttpDownloader(Settings settings) {
067 downloader = new BaseHttpDownloader(settings.getProperties(), null);
068 }
069
070 @Override
071 String description(URI uri) {
072 return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
073 }
074
075 @Override
076 String[] getSupportedSchemes() {
077 return new String[]{"http", "https"};
078 }
079
080 @Override
081 byte[] readBytes(URI uri) {
082 return download(uri);
083 }
084
085 @Override
086 String readString(URI uri, Charset charset) {
087 try {
088 return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri), charset));
089 } catch (IOException e) {
090 throw failToDownload(uri, e);
091 }
092 }
093
094 public String downloadPlainText(URI uri, String encoding) {
095 return readString(uri, Charset.forName(encoding));
096 }
097
098 public byte[] download(URI uri) {
099 try {
100 return ByteStreams.toByteArray(downloader.newInputSupplier(uri));
101 } catch (IOException e) {
102 throw failToDownload(uri, e);
103 }
104 }
105
106 public String getProxySynthesis(URI uri) {
107 return downloader.getProxySynthesis(uri);
108 }
109
110 public InputStream openStream(URI uri) {
111 try {
112 return downloader.newInputSupplier(uri).getInput();
113 } catch (IOException e) {
114 throw failToDownload(uri, e);
115 }
116 }
117
118 public void download(URI uri, File toFile) {
119 try {
120 Files.copy(downloader.newInputSupplier(uri), toFile);
121 } catch (IOException e) {
122 FileUtils.deleteQuietly(toFile);
123 throw failToDownload(uri, e);
124 }
125 }
126
127 private SonarException failToDownload(URI uri, IOException e) {
128 throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
129 }
130
131 public static class BaseHttpDownloader {
132 private static final List<String> PROXY_SETTINGS = ImmutableList.of(
133 "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
134 "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
135
136 private String userAgent;
137
138 public BaseHttpDownloader(Map<String, String> settings, String userAgent) {
139 initProxy(settings);
140 initUserAgent(userAgent);
141 }
142
143 private void initProxy(Map<String, String> settings) {
144 propagateProxySystemProperties(settings);
145 if (requiresProxyAuthentication(settings)) {
146 registerProxyCredentials(settings);
147 }
148 }
149
150 private void initUserAgent(String sonarVersion) {
151 userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion));
152 System.setProperty("http.agent", userAgent);
153 }
154
155 private String getProxySynthesis(URI uri) {
156 return getProxySynthesis(uri, ProxySelector.getDefault());
157 }
158
159 @VisibleForTesting
160 static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
161 List<Proxy> proxies = proxySelector.select(uri);
162 if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
163 return "no proxy";
164 }
165
166 List<String> descriptions = Lists.newArrayList();
167 for (Proxy proxy : proxies) {
168 if (proxy.type() != Proxy.Type.DIRECT) {
169 descriptions.add("proxy: " + proxy.address());
170 }
171 }
172
173 return Joiner.on(", ").join(descriptions);
174 }
175
176 private void registerProxyCredentials(Map<String, String> settings) {
177 Authenticator.setDefault(new ProxyAuthenticator(
178 settings.get("http.proxyUser"),
179 settings.get("http.proxyPassword")));
180 }
181
182 private boolean requiresProxyAuthentication(Map<String, String> settings) {
183 return settings.containsKey("http.proxyUser");
184 }
185
186 private void propagateProxySystemProperties(Map<String, String> settings) {
187 for (String key : PROXY_SETTINGS) {
188 if (settings.containsKey(key)) {
189 System.setProperty(key, settings.get(key));
190 }
191 }
192 }
193
194 public InputSupplier<InputStream> newInputSupplier(URI uri) {
195 return new HttpInputSupplier(uri, userAgent, null, null);
196 }
197
198 public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
199 return new HttpInputSupplier(uri, userAgent, login, password);
200 }
201
202 private static class HttpInputSupplier implements InputSupplier<InputStream> {
203 private final String login;
204 private final String password;
205 private final URI uri;
206 private final String userAgent;
207
208 HttpInputSupplier(URI uri, String userAgent, String login, String password) {
209 this.uri = uri;
210 this.userAgent = userAgent;
211 this.login = login;
212 this.password = password;
213 }
214
215 public InputStream getInput() throws IOException {
216 LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
217
218 HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
219 if (!Strings.isNullOrEmpty(login)) {
220 String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes()));
221 connection.setRequestProperty("Authorization", "Basic " + encoded);
222 }
223 connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
224 connection.setReadTimeout(TIMEOUT_MILLISECONDS);
225 connection.setUseCaches(true);
226 connection.setInstanceFollowRedirects(true);
227 connection.setRequestProperty("User-Agent", userAgent);
228
229 int responseCode = connection.getResponseCode();
230 if (responseCode >= 400) {
231 throw new HttpException(uri, responseCode);
232 }
233
234 return connection.getInputStream();
235 }
236 }
237
238 private static class ProxyAuthenticator extends Authenticator {
239 private final PasswordAuthentication auth;
240
241 ProxyAuthenticator(String user, String password) {
242 auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
243 }
244
245 @Override
246 protected PasswordAuthentication getPasswordAuthentication() {
247 return auth;
248 }
249 }
250 }
251
252 public static class HttpException extends RuntimeException {
253 private final URI uri;
254 private final int responseCode;
255
256 public HttpException(URI uri, int responseCode) {
257 super("Fail to download [" + uri + "]. Response code: " + responseCode);
258 this.uri = uri;
259 this.responseCode = responseCode;
260 }
261
262 public int getResponseCode() {
263 return responseCode;
264 }
265
266 public URI getUri() {
267 return uri;
268 }
269 }
270 }