001 /* 002 * SonarQube, open source software quality management tool. 003 * Copyright (C) 2008-2014 SonarSource 004 * mailto:contact AT sonarsource DOT com 005 * 006 * SonarQube is free software; you can redistribute it and/or 007 * modify it under the terms of the GNU Lesser General Public 008 * License as published by the Free Software Foundation; either 009 * version 3 of the License, or (at your option) any later version. 010 * 011 * SonarQube is distributed in the hope that it will be useful, 012 * but WITHOUT ANY WARRANTY; without even the implied warranty of 013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 014 * Lesser General Public License for more details. 015 * 016 * You should have received a copy of the GNU Lesser General Public License 017 * along with this program; if not, write to the Free Software Foundation, 018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 019 */ 020 package org.sonar.search; 021 022 import org.apache.commons.lang.StringUtils; 023 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; 024 import org.elasticsearch.common.settings.ImmutableSettings; 025 import org.elasticsearch.common.unit.TimeValue; 026 import org.elasticsearch.node.Node; 027 import org.elasticsearch.node.NodeBuilder; 028 import org.slf4j.LoggerFactory; 029 import org.sonar.process.MinimumViableSystem; 030 import org.sonar.process.Monitored; 031 import org.sonar.process.ProcessEntryPoint; 032 import org.sonar.process.ProcessLogging; 033 import org.sonar.process.Props; 034 import org.sonar.search.script.ListUpdate; 035 036 import java.io.File; 037 import java.net.InetAddress; 038 import java.util.Collections; 039 import java.util.HashSet; 040 import java.util.Set; 041 042 public class SearchServer implements Monitored { 043 044 public static final String SONAR_NODE_NAME = "sonar.node.name"; 045 public static final String ES_PORT_PROPERTY = "sonar.search.port"; 046 public static final String ES_CLUSTER_PROPERTY = "sonar.cluster.name"; 047 public static final String ES_CLUSTER_INET = "sonar.cluster.master"; 048 049 public static final String SONAR_PATH_HOME = "sonar.path.home"; 050 public static final String SONAR_PATH_DATA = "sonar.path.data"; 051 public static final String SONAR_PATH_TEMP = "sonar.path.temp"; 052 public static final String SONAR_PATH_LOG = "sonar.path.log"; 053 054 private static final Integer MINIMUM_INDEX_REPLICATION = 1; 055 056 private final Set<String> nodes = new HashSet<String>(); 057 private final Props props; 058 059 private Node node; 060 061 public SearchServer(Props props) { 062 this.props = props; 063 new MinimumViableSystem().check(); 064 065 String esNodesInets = props.value(ES_CLUSTER_INET); 066 if (StringUtils.isNotEmpty(esNodesInets)) { 067 Collections.addAll(nodes, esNodesInets.split(",")); 068 } 069 } 070 071 @Override 072 public void start() { 073 Integer port = props.valueAsInt(ES_PORT_PROPERTY); 074 String clusterName = props.value(ES_CLUSTER_PROPERTY); 075 076 LoggerFactory.getLogger(SearchServer.class).info("Starting ES[{}] on port: {}", clusterName, port); 077 078 ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder() 079 080 // Disable MCast 081 .put("discovery.zen.ping.multicast.enabled", "false") 082 083 // Index storage policies 084 .put("index.merge.policy.max_merge_at_once", "200") 085 .put("index.merge.policy.segments_per_tier", "200") 086 .put("index.number_of_shards", "1") 087 .put("index.number_of_replicas", MINIMUM_INDEX_REPLICATION) 088 .put("index.store.type", "mmapfs") 089 .put("indices.store.throttle.type", "merge") 090 .put("indices.store.throttle.max_bytes_per_sec", "200mb") 091 092 // Install our own listUpdate scripts 093 .put("script.default_lang", "native") 094 .put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName()) 095 096 // Node is pure transport 097 .put("transport.tcp.port", port) 098 .put("http.enabled", false) 099 100 // Setting up ES paths 101 .put("path.data", esDataDir().getAbsolutePath()) 102 .put("path.work", esWorkDir().getAbsolutePath()) 103 .put("path.logs", esLogDir().getAbsolutePath()); 104 105 if (!nodes.isEmpty()) { 106 107 LoggerFactory.getLogger(SearchServer.class).info("Joining ES cluster with master: {}", nodes); 108 esSettings.put("discovery.zen.ping.unicast.hosts", StringUtils.join(nodes, ",")); 109 esSettings.put("node.master", false); 110 // Enforce a N/2+1 number of masters in cluster 111 esSettings.put("discovery.zen.minimum_master_nodes", 1); 112 // Change master pool requirement when in distributed mode 113 // esSettings.put("discovery.zen.minimum_master_nodes", (int) Math.floor(nodes.size() / 2.0) + 1); 114 } 115 116 // Set cluster coordinates 117 esSettings.put("cluster.name", clusterName); 118 esSettings.put("node.rack_id", props.value(SONAR_NODE_NAME, "unknown")); 119 esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id"); 120 if (props.contains(SONAR_NODE_NAME)) { 121 esSettings.put("node.name", props.value(SONAR_NODE_NAME)); 122 } else { 123 try { 124 esSettings.put("node.name", InetAddress.getLocalHost().getHostName()); 125 } catch (Exception e) { 126 LoggerFactory.getLogger(SearchServer.class).warn("Could not determine hostname", e); 127 esSettings.put("node.name", "sq-" + System.currentTimeMillis()); 128 } 129 } 130 131 // Make sure the index settings are up to date. 132 initAnalysis(esSettings); 133 134 // And building our ES Node 135 node = NodeBuilder.nodeBuilder() 136 .settings(esSettings) 137 .build().start(); 138 139 node.client().admin().indices() 140 .preparePutTemplate("default") 141 .setTemplate("*") 142 .addMapping("_default_", "{\"dynamic\": \"strict\"}") 143 .get(); 144 } 145 146 @Override 147 public boolean isReady() { 148 return node != null && node.client().admin().cluster().prepareHealth() 149 .setWaitForYellowStatus() 150 .setTimeout(TimeValue.timeValueSeconds(3L)) 151 .get() 152 .getStatus() != ClusterHealthStatus.RED; 153 } 154 155 @Override 156 public void awaitStop() { 157 while (node != null && !node.isClosed()) { 158 try { 159 Thread.sleep(200L); 160 } catch (InterruptedException e) { 161 // Ignore 162 } 163 } 164 } 165 166 private void initAnalysis(ImmutableSettings.Builder esSettings) { 167 esSettings 168 169 // Disallow dynamic mapping (too expensive) 170 .put("index.mapper.dynamic", false) 171 172 // Sortable text analyzer 173 .put("index.analysis.analyzer.sortable.type", "custom") 174 .put("index.analysis.analyzer.sortable.tokenizer", "keyword") 175 .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") 176 177 // Edge NGram index-analyzer 178 .put("index.analysis.analyzer.index_grams.type", "custom") 179 .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") 180 .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") 181 182 // Edge NGram search-analyzer 183 .put("index.analysis.analyzer.search_grams.type", "custom") 184 .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") 185 .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") 186 187 // Word index-analyzer 188 .put("index.analysis.analyzer.index_words.type", "custom") 189 .put("index.analysis.analyzer.index_words.tokenizer", "standard") 190 .putArray("index.analysis.analyzer.index_words.filter", 191 "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") 192 193 // Word search-analyzer 194 .put("index.analysis.analyzer.search_words.type", "custom") 195 .put("index.analysis.analyzer.search_words.tokenizer", "standard") 196 .putArray("index.analysis.analyzer.search_words.filter", 197 "standard", "lowercase", "stop", "asciifolding", "porter_stem") 198 199 // Edge NGram filter 200 .put("index.analysis.filter.gram_filter.type", "edgeNGram") 201 .put("index.analysis.filter.gram_filter.min_gram", 2) 202 .put("index.analysis.filter.gram_filter.max_gram", 15) 203 .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") 204 205 // Word filter 206 .put("index.analysis.filter.word_filter.type", "word_delimiter") 207 .put("index.analysis.filter.word_filter.generate_word_parts", true) 208 .put("index.analysis.filter.word_filter.catenate_words", true) 209 .put("index.analysis.filter.word_filter.catenate_numbers", true) 210 .put("index.analysis.filter.word_filter.catenate_all", true) 211 .put("index.analysis.filter.word_filter.split_on_case_change", true) 212 .put("index.analysis.filter.word_filter.preserve_original", true) 213 .put("index.analysis.filter.word_filter.split_on_numerics", true) 214 .put("index.analysis.filter.word_filter.stem_english_possessive", true) 215 216 // Path Analyzer 217 .put("index.analysis.analyzer.path_analyzer.type", "custom") 218 .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); 219 220 } 221 222 private File esHomeDir() { 223 return props.nonNullValueAsFile(SONAR_PATH_HOME); 224 } 225 226 private File esDataDir() { 227 String dataDir = props.value(SONAR_PATH_DATA); 228 if (StringUtils.isNotEmpty(dataDir)) { 229 return new File(dataDir, "es"); 230 } 231 return new File(esHomeDir(), "data/es"); 232 } 233 234 private File esLogDir() { 235 String logDir = props.value(SONAR_PATH_LOG); 236 if (StringUtils.isNotEmpty(logDir)) { 237 return new File(logDir); 238 } 239 return new File(esHomeDir(), "log"); 240 } 241 242 private File esWorkDir() { 243 String workDir = props.value(SONAR_PATH_TEMP); 244 if (StringUtils.isNotEmpty(workDir)) { 245 return new File(workDir); 246 } 247 return new File(esHomeDir(), "temp"); 248 } 249 250 @Override 251 public void stop() { 252 if (node != null && !node.isClosed()) { 253 node.close(); 254 } 255 } 256 257 public static void main(String... args) { 258 ProcessEntryPoint entryPoint = ProcessEntryPoint.createForArguments(args); 259 new ProcessLogging().configure(entryPoint.getProps(), "/org/sonar/search/logback.xml"); 260 SearchServer searchServer = new SearchServer(entryPoint.getProps()); 261 entryPoint.launch(searchServer); 262 } 263 }