001/*
002 * Sonar, open source software quality management tool.
003 * Copyright (C) 2008-2012 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * Sonar is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * Sonar is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public
017 * License along with Sonar; if not, write to the Free Software
018 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
019 */
020package org.sonar.plugins.cpd;
021
022import com.google.common.collect.Iterables;
023import org.apache.commons.io.IOUtils;
024import org.slf4j.Logger;
025import org.slf4j.LoggerFactory;
026import org.sonar.api.batch.SensorContext;
027import org.sonar.api.database.model.ResourceModel;
028import org.sonar.api.measures.CoreMetrics;
029import org.sonar.api.measures.Measure;
030import org.sonar.api.measures.PersistenceMode;
031import org.sonar.api.resources.*;
032import org.sonar.api.utils.SonarException;
033import org.sonar.duplications.block.Block;
034import org.sonar.duplications.block.BlockChunker;
035import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm;
036import org.sonar.duplications.index.CloneGroup;
037import org.sonar.duplications.index.CloneIndex;
038import org.sonar.duplications.index.ClonePart;
039import org.sonar.duplications.java.JavaStatementBuilder;
040import org.sonar.duplications.java.JavaTokenProducer;
041import org.sonar.duplications.statement.Statement;
042import org.sonar.duplications.statement.StatementChunker;
043import org.sonar.duplications.token.TokenChunker;
044import org.sonar.plugins.cpd.index.IndexFactory;
045import org.sonar.plugins.cpd.index.SonarDuplicationsIndex;
046
047import javax.annotation.Nullable;
048
049import java.io.FileInputStream;
050import java.io.FileNotFoundException;
051import java.io.InputStreamReader;
052import java.io.Reader;
053import java.util.Collection;
054import java.util.HashSet;
055import java.util.List;
056import java.util.Set;
057import java.util.concurrent.*;
058
059public class SonarEngine extends CpdEngine {
060
061  private static final Logger LOG = LoggerFactory.getLogger(SonarEngine.class);
062
063  private static final int BLOCK_SIZE = 10;
064
065  /**
066   * Limit of time to analyse one file (in seconds).
067   */
068  private static final int TIMEOUT = 5 * 60;
069
070  private final IndexFactory indexFactory;
071
072  public SonarEngine(IndexFactory indexFactory) {
073    this.indexFactory = indexFactory;
074  }
075
076  @Override
077  public boolean isLanguageSupported(Language language) {
078    return Java.INSTANCE.equals(language);
079  }
080
081  static String getFullKey(Project project, Resource resource) {
082    return new StringBuilder(ResourceModel.KEY_SIZE)
083        .append(project.getKey())
084        .append(':')
085        .append(resource.getKey())
086        .toString();
087  }
088
089  @Override
090  public void analyse(Project project, SensorContext context) {
091    List<InputFile> inputFiles = project.getFileSystem().mainFiles(project.getLanguageKey());
092    if (inputFiles.isEmpty()) {
093      return;
094    }
095    SonarDuplicationsIndex index = createIndex(project, inputFiles);
096    detect(index, context, project, inputFiles);
097  }
098
099  private SonarDuplicationsIndex createIndex(Project project, List<InputFile> inputFiles) {
100    final SonarDuplicationsIndex index = indexFactory.create(project);
101
102    TokenChunker tokenChunker = JavaTokenProducer.build();
103    StatementChunker statementChunker = JavaStatementBuilder.build();
104    BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
105
106    for (InputFile inputFile : inputFiles) {
107      LOG.debug("Populating index from {}", inputFile.getFile());
108      Resource resource = getResource(inputFile);
109      String resourceKey = getFullKey(project, resource);
110
111      List<Statement> statements;
112
113      Reader reader = null;
114      try {
115        reader = new InputStreamReader(new FileInputStream(inputFile.getFile()), project.getFileSystem().getSourceCharset());
116        statements = statementChunker.chunk(tokenChunker.chunk(reader));
117      } catch (FileNotFoundException e) {
118        throw new SonarException(e);
119      } finally {
120        IOUtils.closeQuietly(reader);
121      }
122
123      List<Block> blocks = blockChunker.chunk(resourceKey, statements);
124      index.insert(resource, blocks);
125    }
126
127    return index;
128  }
129
130  private void detect(SonarDuplicationsIndex index, SensorContext context, Project project, List<InputFile> inputFiles) {
131    ExecutorService executorService = Executors.newSingleThreadExecutor();
132    try {
133      for (InputFile inputFile : inputFiles) {
134        LOG.debug("Detection of duplications for {}", inputFile.getFile());
135        Resource resource = getResource(inputFile);
136        String resourceKey = getFullKey(project, resource);
137
138        Collection<Block> fileBlocks = index.getByResource(resource, resourceKey);
139
140        List<CloneGroup> clones;
141        try {
142          clones = executorService.submit(new Task(index, fileBlocks)).get(TIMEOUT, TimeUnit.SECONDS);
143        } catch (TimeoutException e) {
144          clones = null;
145          LOG.warn("Timeout during detection of duplications for " + inputFile.getFile(), e);
146        } catch (InterruptedException e) {
147          throw new SonarException(e);
148        } catch (ExecutionException e) {
149          throw new SonarException(e);
150        }
151
152        save(context, resource, clones);
153      }
154    } finally {
155      executorService.shutdown();
156    }
157  }
158
159  static class Task implements Callable<List<CloneGroup>> {
160    private final CloneIndex index;
161    private final Collection<Block> fileBlocks;
162
163    public Task(CloneIndex index, Collection<Block> fileBlocks) {
164      this.index = index;
165      this.fileBlocks = fileBlocks;
166    }
167
168    public List<CloneGroup> call() {
169      return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks);
170    }
171  }
172
173  private Resource getResource(InputFile inputFile) {
174    return JavaFile.fromRelativePath(inputFile.getRelativePath(), false);
175  }
176
177  static void save(SensorContext context, Resource resource, @Nullable Iterable<CloneGroup> duplications) {
178    if (duplications == null || Iterables.isEmpty(duplications)) {
179      return;
180    }
181    // Calculate number of lines and blocks
182    Set<Integer> duplicatedLines = new HashSet<Integer>();
183    double duplicatedBlocks = 0;
184    for (CloneGroup clone : duplications) {
185      ClonePart origin = clone.getOriginPart();
186      for (ClonePart part : clone.getCloneParts()) {
187        if (part.getResourceId().equals(origin.getResourceId())) {
188          duplicatedBlocks++;
189          for (int duplicatedLine = part.getStartLine(); duplicatedLine < part.getStartLine() + part.getLines(); duplicatedLine++) {
190            duplicatedLines.add(duplicatedLine);
191          }
192        }
193      }
194    }
195    // Save
196    context.saveMeasure(resource, CoreMetrics.DUPLICATED_FILES, 1.0);
197    context.saveMeasure(resource, CoreMetrics.DUPLICATED_LINES, (double) duplicatedLines.size());
198    context.saveMeasure(resource, CoreMetrics.DUPLICATED_BLOCKS, duplicatedBlocks);
199
200    Measure data = new Measure(CoreMetrics.DUPLICATIONS_DATA, toXml(duplications))
201        .setPersistenceMode(PersistenceMode.DATABASE);
202    context.saveMeasure(resource, data);
203  }
204
205  private static String toXml(Iterable<CloneGroup> duplications) {
206    StringBuilder xml = new StringBuilder();
207    xml.append("<duplications>");
208    for (CloneGroup duplication : duplications) {
209      xml.append("<g>");
210      for (ClonePart part : duplication.getCloneParts()) {
211        xml.append("<b s=\"").append(part.getStartLine())
212            .append("\" l=\"").append(part.getLines())
213            .append("\" r=\"").append(part.getResourceId())
214            .append("\"/>");
215      }
216      xml.append("</g>");
217    }
218    xml.append("</duplications>");
219    return xml.toString();
220  }
221
222}