001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.search;
021
022 import org.apache.commons.lang.StringUtils;
023 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
024 import org.elasticsearch.common.settings.ImmutableSettings;
025 import org.elasticsearch.common.unit.TimeValue;
026 import org.elasticsearch.node.Node;
027 import org.elasticsearch.node.NodeBuilder;
028 import org.slf4j.LoggerFactory;
029 import org.sonar.process.MinimumViableSystem;
030 import org.sonar.process.Monitored;
031 import org.sonar.process.ProcessEntryPoint;
032 import org.sonar.process.ProcessLogging;
033 import org.sonar.process.Props;
034 import org.sonar.search.script.ListUpdate;
035
036 import java.io.File;
037 import java.net.InetAddress;
038 import java.util.Collections;
039 import java.util.HashSet;
040 import java.util.Set;
041
042 public class SearchServer implements Monitored {
043
044 public static final String SONAR_NODE_NAME = "sonar.node.name";
045 public static final String ES_PORT_PROPERTY = "sonar.search.port";
046 public static final String ES_CLUSTER_PROPERTY = "sonar.cluster.name";
047 public static final String ES_CLUSTER_INET = "sonar.cluster.master";
048
049 public static final String SONAR_PATH_HOME = "sonar.path.home";
050 public static final String SONAR_PATH_DATA = "sonar.path.data";
051 public static final String SONAR_PATH_TEMP = "sonar.path.temp";
052 public static final String SONAR_PATH_LOG = "sonar.path.log";
053
054 private static final Integer MINIMUM_INDEX_REPLICATION = 1;
055
056 private final Set<String> nodes = new HashSet<String>();
057 private final Props props;
058
059 private Node node;
060
061 public SearchServer(Props props) {
062 this.props = props;
063 new MinimumViableSystem().check();
064
065 String esNodesInets = props.value(ES_CLUSTER_INET);
066 if (StringUtils.isNotEmpty(esNodesInets)) {
067 Collections.addAll(nodes, esNodesInets.split(","));
068 }
069 }
070
071 @Override
072 public void start() {
073 Integer port = props.valueAsInt(ES_PORT_PROPERTY);
074 String clusterName = props.value(ES_CLUSTER_PROPERTY);
075
076 LoggerFactory.getLogger(SearchServer.class).info("Starting ES[{}] on port: {}", clusterName, port);
077
078 ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder()
079
080 // Disable MCast
081 .put("discovery.zen.ping.multicast.enabled", "false")
082
083 // Index storage policies
084 .put("index.merge.policy.max_merge_at_once", "200")
085 .put("index.merge.policy.segments_per_tier", "200")
086 .put("index.number_of_shards", "1")
087 .put("index.number_of_replicas", MINIMUM_INDEX_REPLICATION)
088 .put("index.store.type", "mmapfs")
089 .put("indices.store.throttle.type", "merge")
090 .put("indices.store.throttle.max_bytes_per_sec", "200mb")
091
092 // Install our own listUpdate scripts
093 .put("script.default_lang", "native")
094 .put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName())
095
096 // Node is pure transport
097 .put("transport.tcp.port", port)
098 .put("http.enabled", false)
099
100 // Setting up ES paths
101 .put("path.data", esDataDir().getAbsolutePath())
102 .put("path.work", esWorkDir().getAbsolutePath())
103 .put("path.logs", esLogDir().getAbsolutePath());
104
105 if (!nodes.isEmpty()) {
106
107 LoggerFactory.getLogger(SearchServer.class).info("Joining ES cluster with master: {}", nodes);
108 esSettings.put("discovery.zen.ping.unicast.hosts", StringUtils.join(nodes, ","));
109 esSettings.put("node.master", false);
110 // Enforce a N/2+1 number of masters in cluster
111 esSettings.put("discovery.zen.minimum_master_nodes", 1);
112 // Change master pool requirement when in distributed mode
113 // esSettings.put("discovery.zen.minimum_master_nodes", (int) Math.floor(nodes.size() / 2.0) + 1);
114 }
115
116 // Set cluster coordinates
117 esSettings.put("cluster.name", clusterName);
118 esSettings.put("node.rack_id", props.value(SONAR_NODE_NAME, "unknown"));
119 esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id");
120 if (props.contains(SONAR_NODE_NAME)) {
121 esSettings.put("node.name", props.value(SONAR_NODE_NAME));
122 } else {
123 try {
124 esSettings.put("node.name", InetAddress.getLocalHost().getHostName());
125 } catch (Exception e) {
126 LoggerFactory.getLogger(SearchServer.class).warn("Could not determine hostname", e);
127 esSettings.put("node.name", "sq-" + System.currentTimeMillis());
128 }
129 }
130
131 // Make sure the index settings are up to date.
132 initAnalysis(esSettings);
133
134 // And building our ES Node
135 node = NodeBuilder.nodeBuilder()
136 .settings(esSettings)
137 .build().start();
138
139 node.client().admin().indices()
140 .preparePutTemplate("default")
141 .setTemplate("*")
142 .addMapping("_default_", "{\"dynamic\": \"strict\"}")
143 .get();
144 }
145
146 @Override
147 public boolean isReady() {
148 return node.client().admin().cluster().prepareHealth()
149 .setWaitForYellowStatus()
150 .setTimeout(TimeValue.timeValueSeconds(3L))
151 .get()
152 .getStatus() != ClusterHealthStatus.RED;
153 }
154
155 @Override
156 public void awaitStop() {
157 while (node != null && !node.isClosed()) {
158 try {
159 Thread.sleep(200L);
160 } catch (InterruptedException e) {
161 // Ignore
162 }
163 }
164 }
165
166 private void initAnalysis(ImmutableSettings.Builder esSettings) {
167 esSettings
168
169 // Disallow dynamic mapping (too expensive)
170 .put("index.mapper.dynamic", false)
171
172 // Sortable text analyzer
173 .put("index.analysis.analyzer.sortable.type", "custom")
174 .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
175 .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate")
176
177 // Edge NGram index-analyzer
178 .put("index.analysis.analyzer.index_grams.type", "custom")
179 .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
180 .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
181
182 // Edge NGram search-analyzer
183 .put("index.analysis.analyzer.search_grams.type", "custom")
184 .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
185 .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
186
187 // Word index-analyzer
188 .put("index.analysis.analyzer.index_words.type", "custom")
189 .put("index.analysis.analyzer.index_words.tokenizer", "standard")
190 .putArray("index.analysis.analyzer.index_words.filter",
191 "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
192
193 // Word search-analyzer
194 .put("index.analysis.analyzer.search_words.type", "custom")
195 .put("index.analysis.analyzer.search_words.tokenizer", "standard")
196 .putArray("index.analysis.analyzer.search_words.filter",
197 "standard", "lowercase", "stop", "asciifolding", "porter_stem")
198
199 // Edge NGram filter
200 .put("index.analysis.filter.gram_filter.type", "edgeNGram")
201 .put("index.analysis.filter.gram_filter.min_gram", 2)
202 .put("index.analysis.filter.gram_filter.max_gram", 15)
203 .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
204
205 // Word filter
206 .put("index.analysis.filter.word_filter.type", "word_delimiter")
207 .put("index.analysis.filter.word_filter.generate_word_parts", true)
208 .put("index.analysis.filter.word_filter.catenate_words", true)
209 .put("index.analysis.filter.word_filter.catenate_numbers", true)
210 .put("index.analysis.filter.word_filter.catenate_all", true)
211 .put("index.analysis.filter.word_filter.split_on_case_change", true)
212 .put("index.analysis.filter.word_filter.preserve_original", true)
213 .put("index.analysis.filter.word_filter.split_on_numerics", true)
214 .put("index.analysis.filter.word_filter.stem_english_possessive", true)
215
216 // Path Analyzer
217 .put("index.analysis.analyzer.path_analyzer.type", "custom")
218 .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy");
219
220 }
221
222 private File esHomeDir() {
223 return props.nonNullValueAsFile(SONAR_PATH_HOME);
224 }
225
226 private File esDataDir() {
227 String dataDir = props.value(SONAR_PATH_DATA);
228 if (StringUtils.isNotEmpty(dataDir)) {
229 return new File(dataDir, "es");
230 }
231 return new File(esHomeDir(), "data/es");
232 }
233
234 private File esLogDir() {
235 String logDir = props.value(SONAR_PATH_LOG);
236 if (StringUtils.isNotEmpty(logDir)) {
237 return new File(logDir);
238 }
239 return new File(esHomeDir(), "log");
240 }
241
242 private File esWorkDir() {
243 String workDir = props.value(SONAR_PATH_TEMP);
244 if (StringUtils.isNotEmpty(workDir)) {
245 return new File(workDir);
246 }
247 return new File(esHomeDir(), "temp");
248 }
249
250 @Override
251 public synchronized void stop() {
252 if (node != null && !node.isClosed()) {
253 node.close();
254 }
255 }
256
257 public static void main(String... args) {
258 ProcessEntryPoint entryPoint = ProcessEntryPoint.createForArguments(args);
259 new ProcessLogging().configure(entryPoint.getProps(), "/org/sonar/search/logback.xml");
260 SearchServer searchServer = new SearchServer(entryPoint.getProps());
261 entryPoint.launch(searchServer);
262 }
263 }