001/* 002 * SonarQube, open source software quality management tool. 003 * Copyright (C) 2008-2014 SonarSource 004 * mailto:contact AT sonarsource DOT com 005 * 006 * SonarQube is free software; you can redistribute it and/or 007 * modify it under the terms of the GNU Lesser General Public 008 * License as published by the Free Software Foundation; either 009 * version 3 of the License, or (at your option) any later version. 010 * 011 * SonarQube is distributed in the hope that it will be useful, 012 * but WITHOUT ANY WARRANTY; without even the implied warranty of 013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 014 * Lesser General Public License for more details. 015 * 016 * You should have received a copy of the GNU Lesser General Public License 017 * along with this program; if not, write to the Free Software Foundation, 018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 019 */ 020package org.sonar.xoo.lang; 021 022import com.google.common.base.Splitter; 023import net.sourceforge.pmd.cpd.SourceCode; 024import net.sourceforge.pmd.cpd.TokenEntry; 025import net.sourceforge.pmd.cpd.Tokenizer; 026import net.sourceforge.pmd.cpd.Tokens; 027import org.apache.commons.io.FileUtils; 028import org.sonar.api.BatchComponent; 029import org.sonar.api.batch.fs.FileSystem; 030 031import java.io.File; 032import java.io.IOException; 033 034public class XooTokenizer implements Tokenizer, BatchComponent { 035 036 private FileSystem fs; 037 038 public XooTokenizer(FileSystem fs) { 039 this.fs = fs; 040 } 041 042 @Override 043 public final void tokenize(SourceCode source, Tokens cpdTokens) { 044 String fileName = source.getFileName(); 045 int lineIdx = 1; 046 try { 047 for (String line : FileUtils.readLines(new File(fileName), fs.encoding())) { 048 for (String token : Splitter.on(" ").split(line)) { 049 TokenEntry cpdToken = new TokenEntry(token, fileName, lineIdx); 050 cpdTokens.add(cpdToken); 051 } 052 lineIdx++; 053 } 054 } catch (IOException e) { 055 throw new IllegalStateException("Unable to tokenize", e); 056 } 057 cpdTokens.add(TokenEntry.getEOF()); 058 } 059}