001 /*
002 * Sonar, open source software quality management tool.
003 * Copyright (C) 2008-2012 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * Sonar is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * Sonar is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public
017 * License along with Sonar; if not, write to the Free Software
018 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02
019 */
020 package org.sonar.plugins.cpd;
021
022 import com.google.common.collect.Iterables;
023 import org.apache.commons.io.IOUtils;
024 import org.slf4j.Logger;
025 import org.slf4j.LoggerFactory;
026 import org.sonar.api.batch.SensorContext;
027 import org.sonar.api.database.model.ResourceModel;
028 import org.sonar.api.measures.CoreMetrics;
029 import org.sonar.api.measures.Measure;
030 import org.sonar.api.measures.PersistenceMode;
031 import org.sonar.api.resources.*;
032 import org.sonar.api.utils.SonarException;
033 import org.sonar.duplications.block.Block;
034 import org.sonar.duplications.block.BlockChunker;
035 import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
036 import org.sonar.duplications.index.CloneGroup;
037 import org.sonar.duplications.index.CloneIndex;
038 import org.sonar.duplications.index.ClonePart;
039 import org.sonar.duplications.java.JavaStatementBuilder;
040 import org.sonar.duplications.java.JavaTokenProducer;
041 import org.sonar.duplications.statement.Statement;
042 import org.sonar.duplications.statement.StatementChunker;
043 import org.sonar.duplications.token.TokenChunker;
044 import org.sonar.plugins.cpd.index.IndexFactory;
045 import org.sonar.plugins.cpd.index.SonarDuplicationsIndex;
046
047 import javax.annotation.Nullable;
048
049 import java.io.FileInputStream;
050 import java.io.FileNotFoundException;
051 import java.io.InputStreamReader;
052 import java.io.Reader;
053 import java.util.Collection;
054 import java.util.HashSet;
055 import java.util.List;
056 import java.util.Set;
057 import java.util.concurrent.*;
058
059 public class SonarEngine extends CpdEngine {
060
061 private static final Logger LOG = LoggerFactory.getLogger(SonarEngine.class);
062
063 private static final int BLOCK_SIZE = 10;
064
065 /**
066 * Limit of time to analyse one file (in seconds).
067 */
068 private static final int TIMEOUT = 5 * 60;
069
070 private final IndexFactory indexFactory;
071
072 public SonarEngine(IndexFactory indexFactory) {
073 this.indexFactory = indexFactory;
074 }
075
076 @Override
077 public boolean isLanguageSupported(Language language) {
078 return Java.INSTANCE.equals(language);
079 }
080
081 static String getFullKey(Project project, Resource resource) {
082 return new StringBuilder(ResourceModel.KEY_SIZE)
083 .append(project.getKey())
084 .append(':')
085 .append(resource.getKey())
086 .toString();
087 }
088
089 @Override
090 public void analyse(Project project, SensorContext context) {
091 List<InputFile> inputFiles = project.getFileSystem().mainFiles(project.getLanguageKey());
092 if (inputFiles.isEmpty()) {
093 return;
094 }
095 SonarDuplicationsIndex index = createIndex(project, inputFiles);
096 detect(index, context, project, inputFiles);
097 }
098
099 private SonarDuplicationsIndex createIndex(Project project, List<InputFile> inputFiles) {
100 final SonarDuplicationsIndex index = indexFactory.create(project);
101
102 TokenChunker tokenChunker = JavaTokenProducer.build();
103 StatementChunker statementChunker = JavaStatementBuilder.build();
104 BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
105
106 for (InputFile inputFile : inputFiles) {
107 LOG.debug("Populating index from {}", inputFile.getFile());
108 Resource resource = getResource(inputFile);
109 String resourceKey = getFullKey(project, resource);
110
111 List<Statement> statements;
112
113 Reader reader = null;
114 try {
115 reader = new InputStreamReader(new FileInputStream(inputFile.getFile()), project.getFileSystem().getSourceCharset());
116 statements = statementChunker.chunk(tokenChunker.chunk(reader));
117 } catch (FileNotFoundException e) {
118 throw new SonarException(e);
119 } finally {
120 IOUtils.closeQuietly(reader);
121 }
122
123 List<Block> blocks = blockChunker.chunk(resourceKey, statements);
124 index.insert(resource, blocks);
125 }
126
127 return index;
128 }
129
130 private void detect(SonarDuplicationsIndex index, SensorContext context, Project project, List<InputFile> inputFiles) {
131 ExecutorService executorService = Executors.newSingleThreadExecutor();
132 try {
133 for (InputFile inputFile : inputFiles) {
134 LOG.debug("Detection of duplications for {}", inputFile.getFile());
135 Resource resource = getResource(inputFile);
136 String resourceKey = getFullKey(project, resource);
137
138 Collection<Block> fileBlocks = index.getByResource(resource, resourceKey);
139
140 List<CloneGroup> clones;
141 try {
142 clones = executorService.submit(new Task(index, fileBlocks)).get(TIMEOUT, TimeUnit.SECONDS);
143 } catch (TimeoutException e) {
144 clones = null;
145 LOG.warn("Timeout during detection of duplications for " + inputFile.getFile(), e);
146 } catch (InterruptedException e) {
147 throw new SonarException(e);
148 } catch (ExecutionException e) {
149 throw new SonarException(e);
150 }
151
152 save(context, resource, clones);
153 }
154 } finally {
155 executorService.shutdown();
156 }
157 }
158
159 static class Task implements Callable<List<CloneGroup>> {
160 private final CloneIndex index;
161 private final Collection<Block> fileBlocks;
162
163 public Task(CloneIndex index, Collection<Block> fileBlocks) {
164 this.index = index;
165 this.fileBlocks = fileBlocks;
166 }
167
168 public List<CloneGroup> call() {
169 return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks);
170 }
171 }
172
173 private Resource getResource(InputFile inputFile) {
174 return JavaFile.fromRelativePath(inputFile.getRelativePath(), false);
175 }
176
177 static void save(SensorContext context, Resource resource, @Nullable Iterable<CloneGroup> duplications) {
178 if (duplications == null || Iterables.isEmpty(duplications)) {
179 return;
180 }
181 // Calculate number of lines and blocks
182 Set<Integer> duplicatedLines = new HashSet<Integer>();
183 double duplicatedBlocks = 0;
184 for (CloneGroup clone : duplications) {
185 ClonePart origin = clone.getOriginPart();
186 for (ClonePart part : clone.getCloneParts()) {
187 if (part.getResourceId().equals(origin.getResourceId())) {
188 duplicatedBlocks++;
189 for (int duplicatedLine = part.getStartLine(); duplicatedLine < part.getStartLine() + part.getLines(); duplicatedLine++) {
190 duplicatedLines.add(duplicatedLine);
191 }
192 }
193 }
194 }
195 // Save
196 context.saveMeasure(resource, CoreMetrics.DUPLICATED_FILES, 1.0);
197 context.saveMeasure(resource, CoreMetrics.DUPLICATED_LINES, (double) duplicatedLines.size());
198 context.saveMeasure(resource, CoreMetrics.DUPLICATED_BLOCKS, duplicatedBlocks);
199
200 Measure data = new Measure(CoreMetrics.DUPLICATIONS_DATA, toXml(duplications))
201 .setPersistenceMode(PersistenceMode.DATABASE);
202 context.saveMeasure(resource, data);
203 }
204
205 private static String toXml(Iterable<CloneGroup> duplications) {
206 StringBuilder xml = new StringBuilder();
207 xml.append("<duplications>");
208 for (CloneGroup duplication : duplications) {
209 xml.append("<g>");
210 for (ClonePart part : duplication.getCloneParts()) {
211 xml.append("<b s=\"").append(part.getStartLine())
212 .append("\" l=\"").append(part.getLines())
213 .append("\" r=\"").append(part.getResourceId())
214 .append("\"/>");
215 }
216 xml.append("</g>");
217 }
218 xml.append("</duplications>");
219 return xml.toString();
220 }
221
222 }