001/*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
019 */
020package org.sonar.search;
021
022import org.apache.commons.lang.StringUtils;
023import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
024import org.elasticsearch.common.settings.ImmutableSettings;
025import org.elasticsearch.common.unit.TimeValue;
026import org.elasticsearch.node.Node;
027import org.elasticsearch.node.NodeBuilder;
028import org.slf4j.LoggerFactory;
029import org.sonar.process.MinimumViableSystem;
030import org.sonar.process.Monitored;
031import org.sonar.process.ProcessEntryPoint;
032import org.sonar.process.ProcessLogging;
033import org.sonar.process.Props;
034import org.sonar.search.script.ListUpdate;
035
036import java.io.File;
037import java.net.InetAddress;
038import java.util.Collections;
039import java.util.HashSet;
040import java.util.Set;
041
042public class SearchServer implements Monitored {
043
044  public static final String SONAR_NODE_NAME = "sonar.node.name";
045  public static final String ES_PORT_PROPERTY = "sonar.search.port";
046  public static final String ES_CLUSTER_PROPERTY = "sonar.cluster.name";
047  public static final String ES_CLUSTER_INET = "sonar.cluster.master";
048
049  public static final String SONAR_PATH_HOME = "sonar.path.home";
050  public static final String SONAR_PATH_DATA = "sonar.path.data";
051  public static final String SONAR_PATH_TEMP = "sonar.path.temp";
052  public static final String SONAR_PATH_LOG = "sonar.path.log";
053
054  private static final Integer MINIMUM_INDEX_REPLICATION = 1;
055
056  private final Set<String> nodes = new HashSet<String>();
057  private final Props props;
058
059  private Node node;
060
061  public SearchServer(Props props) {
062    this.props = props;
063    new MinimumViableSystem().check();
064
065    String esNodesInets = props.value(ES_CLUSTER_INET);
066    if (StringUtils.isNotEmpty(esNodesInets)) {
067      Collections.addAll(nodes, esNodesInets.split(","));
068    }
069  }
070
071  @Override
072  public void start() {
073    Integer port = props.valueAsInt(ES_PORT_PROPERTY);
074    String clusterName = props.value(ES_CLUSTER_PROPERTY);
075
076    LoggerFactory.getLogger(SearchServer.class).info("Starting ES[{}] on port: {}", clusterName, port);
077
078    ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder()
079
080      // Disable MCast
081      .put("discovery.zen.ping.multicast.enabled", "false")
082
083      // Index storage policies
084      .put("index.merge.policy.max_merge_at_once", "200")
085      .put("index.merge.policy.segments_per_tier", "200")
086      .put("index.number_of_shards", "1")
087      .put("index.number_of_replicas", MINIMUM_INDEX_REPLICATION)
088      .put("index.store.type", "mmapfs")
089      .put("indices.store.throttle.type", "merge")
090      .put("indices.store.throttle.max_bytes_per_sec", "200mb")
091
092      // Install our own listUpdate scripts
093      .put("script.default_lang", "native")
094      .put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName())
095
096      // Node is pure transport
097      .put("transport.tcp.port", port)
098      .put("http.enabled", false)
099
100      // Setting up ES paths
101      .put("path.data", esDataDir().getAbsolutePath())
102      .put("path.work", esWorkDir().getAbsolutePath())
103      .put("path.logs", esLogDir().getAbsolutePath());
104
105    if (!nodes.isEmpty()) {
106
107      LoggerFactory.getLogger(SearchServer.class).info("Joining ES cluster with master: {}", nodes);
108      esSettings.put("discovery.zen.ping.unicast.hosts", StringUtils.join(nodes, ","));
109      esSettings.put("node.master", false);
110      // Enforce a N/2+1 number of masters in cluster
111      esSettings.put("discovery.zen.minimum_master_nodes", 1);
112      // Change master pool requirement when in distributed mode
113      // esSettings.put("discovery.zen.minimum_master_nodes", (int) Math.floor(nodes.size() / 2.0) + 1);
114    }
115
116    // Set cluster coordinates
117    esSettings.put("cluster.name", clusterName);
118    esSettings.put("node.rack_id", props.value(SONAR_NODE_NAME, "unknown"));
119    esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id");
120    if (props.contains(SONAR_NODE_NAME)) {
121      esSettings.put("node.name", props.value(SONAR_NODE_NAME));
122    } else {
123      try {
124        esSettings.put("node.name", InetAddress.getLocalHost().getHostName());
125      } catch (Exception e) {
126        LoggerFactory.getLogger(SearchServer.class).warn("Could not determine hostname", e);
127        esSettings.put("node.name", "sq-" + System.currentTimeMillis());
128      }
129    }
130
131    // Make sure the index settings are up to date.
132    initAnalysis(esSettings);
133
134    // And building our ES Node
135    node = NodeBuilder.nodeBuilder()
136      .settings(esSettings)
137      .build().start();
138
139    node.client().admin().indices()
140      .preparePutTemplate("default")
141      .setTemplate("*")
142      .addMapping("_default_", "{\"dynamic\": \"strict\"}")
143      .get();
144  }
145
146  @Override
147  public boolean isReady() {
148    return node != null && node.client().admin().cluster().prepareHealth()
149      .setWaitForYellowStatus()
150      .setTimeout(TimeValue.timeValueSeconds(3L))
151      .get()
152      .getStatus() != ClusterHealthStatus.RED;
153  }
154
155  @Override
156  public void awaitStop() {
157    while (node != null && !node.isClosed()) {
158      try {
159        Thread.sleep(200L);
160      } catch (InterruptedException e) {
161        // Ignore
162      }
163    }
164  }
165
166  private void initAnalysis(ImmutableSettings.Builder esSettings) {
167    esSettings
168
169      // Disallow dynamic mapping (too expensive)
170      .put("index.mapper.dynamic", false)
171
172      // Sortable text analyzer
173      .put("index.analysis.analyzer.sortable.type", "custom")
174      .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
175      .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate")
176
177      // Edge NGram index-analyzer
178      .put("index.analysis.analyzer.index_grams.type", "custom")
179      .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
180      .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
181
182      // Edge NGram search-analyzer
183      .put("index.analysis.analyzer.search_grams.type", "custom")
184      .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
185      .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
186
187      // Word index-analyzer
188      .put("index.analysis.analyzer.index_words.type", "custom")
189      .put("index.analysis.analyzer.index_words.tokenizer", "standard")
190      .putArray("index.analysis.analyzer.index_words.filter",
191        "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
192
193      // Word search-analyzer
194      .put("index.analysis.analyzer.search_words.type", "custom")
195      .put("index.analysis.analyzer.search_words.tokenizer", "standard")
196      .putArray("index.analysis.analyzer.search_words.filter",
197        "standard", "lowercase", "stop", "asciifolding", "porter_stem")
198
199      // Edge NGram filter
200      .put("index.analysis.filter.gram_filter.type", "edgeNGram")
201      .put("index.analysis.filter.gram_filter.min_gram", 2)
202      .put("index.analysis.filter.gram_filter.max_gram", 15)
203      .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
204
205      // Word filter
206      .put("index.analysis.filter.word_filter.type", "word_delimiter")
207      .put("index.analysis.filter.word_filter.generate_word_parts", true)
208      .put("index.analysis.filter.word_filter.catenate_words", true)
209      .put("index.analysis.filter.word_filter.catenate_numbers", true)
210      .put("index.analysis.filter.word_filter.catenate_all", true)
211      .put("index.analysis.filter.word_filter.split_on_case_change", true)
212      .put("index.analysis.filter.word_filter.preserve_original", true)
213      .put("index.analysis.filter.word_filter.split_on_numerics", true)
214      .put("index.analysis.filter.word_filter.stem_english_possessive", true)
215
216      // Path Analyzer
217      .put("index.analysis.analyzer.path_analyzer.type", "custom")
218      .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy");
219
220  }
221
222  private File esHomeDir() {
223    return props.nonNullValueAsFile(SONAR_PATH_HOME);
224  }
225
226  private File esDataDir() {
227    String dataDir = props.value(SONAR_PATH_DATA);
228    if (StringUtils.isNotEmpty(dataDir)) {
229      return new File(dataDir, "es");
230    }
231    return new File(esHomeDir(), "data/es");
232  }
233
234  private File esLogDir() {
235    String logDir = props.value(SONAR_PATH_LOG);
236    if (StringUtils.isNotEmpty(logDir)) {
237      return new File(logDir);
238    }
239    return new File(esHomeDir(), "log");
240  }
241
242  private File esWorkDir() {
243    String workDir = props.value(SONAR_PATH_TEMP);
244    if (StringUtils.isNotEmpty(workDir)) {
245      return new File(workDir);
246    }
247    return new File(esHomeDir(), "temp");
248  }
249
250  @Override
251  public void stop() {
252    if (node != null && !node.isClosed()) {
253      node.close();
254    }
255  }
256
257  public static void main(String... args) {
258    ProcessEntryPoint entryPoint = ProcessEntryPoint.createForArguments(args);
259    new ProcessLogging().configure(entryPoint.getProps(), "/org/sonar/search/logback.xml");
260    SearchServer searchServer = new SearchServer(entryPoint.getProps());
261    entryPoint.launch(searchServer);
262  }
263}