001/*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
019 */
020package org.sonar.xoo.lang;
021
022import com.google.common.base.Splitter;
023import net.sourceforge.pmd.cpd.SourceCode;
024import net.sourceforge.pmd.cpd.TokenEntry;
025import net.sourceforge.pmd.cpd.Tokenizer;
026import net.sourceforge.pmd.cpd.Tokens;
027import org.apache.commons.io.FileUtils;
028import org.sonar.api.BatchComponent;
029import org.sonar.api.batch.fs.FileSystem;
030
031import java.io.File;
032import java.io.IOException;
033
034public class XooTokenizer implements Tokenizer, BatchComponent {
035
036  private FileSystem fs;
037
038  public XooTokenizer(FileSystem fs) {
039    this.fs = fs;
040  }
041
042  @Override
043  public final void tokenize(SourceCode source, Tokens cpdTokens) {
044    String fileName = source.getFileName();
045    int lineIdx = 1;
046    try {
047      for (String line : FileUtils.readLines(new File(fileName), fs.encoding())) {
048        for (String token : Splitter.on(" ").split(line)) {
049          TokenEntry cpdToken = new TokenEntry(token, fileName, lineIdx);
050          cpdTokens.add(cpdToken);
051        }
052        lineIdx++;
053      }
054    } catch (IOException e) {
055      throw new IllegalStateException("Unable to tokenize", e);
056    }
057    cpdTokens.add(TokenEntry.getEOF());
058  }
059}