1 /** 2 * Copyright (c) 2008, http://www.snakeyaml.org 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package org.pyyaml; 17 18 import java.io.File; 19 import java.io.FileInputStream; 20 import java.io.FileNotFoundException; 21 import java.io.IOException; 22 import java.io.InputStream; 23 import java.util.ArrayList; 24 import java.util.HashMap; 25 import java.util.List; 26 import java.util.Map; 27 28 import org.yaml.snakeyaml.reader.StreamReader; 29 import org.yaml.snakeyaml.reader.UnicodeReader; 30 import org.yaml.snakeyaml.scanner.Scanner; 31 import org.yaml.snakeyaml.scanner.ScannerImpl; 32 import org.yaml.snakeyaml.tokens.AliasToken; 33 import org.yaml.snakeyaml.tokens.AnchorToken; 34 import org.yaml.snakeyaml.tokens.BlockEndToken; 35 import org.yaml.snakeyaml.tokens.BlockEntryToken; 36 import org.yaml.snakeyaml.tokens.BlockMappingStartToken; 37 import org.yaml.snakeyaml.tokens.BlockSequenceStartToken; 38 import org.yaml.snakeyaml.tokens.DirectiveToken; 39 import org.yaml.snakeyaml.tokens.DocumentEndToken; 40 import org.yaml.snakeyaml.tokens.DocumentStartToken; 41 import org.yaml.snakeyaml.tokens.FlowEntryToken; 42 import org.yaml.snakeyaml.tokens.FlowMappingEndToken; 43 import org.yaml.snakeyaml.tokens.FlowMappingStartToken; 44 import org.yaml.snakeyaml.tokens.FlowSequenceEndToken; 45 import org.yaml.snakeyaml.tokens.FlowSequenceStartToken; 46 import org.yaml.snakeyaml.tokens.KeyToken; 47 import org.yaml.snakeyaml.tokens.ScalarToken; 48 import org.yaml.snakeyaml.tokens.StreamEndToken; 49 import org.yaml.snakeyaml.tokens.StreamStartToken; 50 import org.yaml.snakeyaml.tokens.TagToken; 51 import org.yaml.snakeyaml.tokens.Token; 52 import org.yaml.snakeyaml.tokens.ValueToken; 53 54 /** 55 * imported from PyYAML 56 */ 57 public class PyTokensTest extends PyImportTest { 58 59 public void testTokens() throws FileNotFoundException { 60 Map<Class<?>, String> replaces = new HashMap<Class<?>, String>(); 61 replaces.put(DirectiveToken.class, "%"); 62 replaces.put(DocumentStartToken.class, "---"); 63 replaces.put(DocumentEndToken.class, "..."); 64 replaces.put(AliasToken.class, "*"); 65 replaces.put(AnchorToken.class, "&"); 66 replaces.put(TagToken.class, "!"); 67 replaces.put(ScalarToken.class, "_"); 68 replaces.put(BlockSequenceStartToken.class, "[["); 69 replaces.put(BlockMappingStartToken.class, "{{"); 70 replaces.put(BlockEndToken.class, "]}"); 71 replaces.put(FlowSequenceStartToken.class, "["); 72 replaces.put(FlowSequenceEndToken.class, "]"); 73 replaces.put(FlowMappingStartToken.class, "{"); 74 replaces.put(FlowMappingEndToken.class, "}"); 75 replaces.put(BlockEntryToken.class, ","); 76 replaces.put(FlowEntryToken.class, ","); 77 replaces.put(KeyToken.class, "?"); 78 replaces.put(ValueToken.class, ":"); 79 // 80 File[] tokensFiles = getStreamsByExtension(".tokens"); 81 assertTrue("No test files found.", tokensFiles.length > 0); 82 for (int i = 0; i < tokensFiles.length; i++) { 83 String name = tokensFiles[i].getName(); 84 int position = name.lastIndexOf('.'); 85 String dataName = name.substring(0, position) + ".data"; 86 // 87 String tokenFileData = getResource(name); 88 String[] split = tokenFileData.split("\\s+"); 89 List<String> tokens2 = new ArrayList<String>(); 90 for (int j = 0; j < split.length; j++) { 91 tokens2.add(split[j]); 92 } 93 // 94 List<String> tokens1 = new ArrayList<String>(); 95 StreamReader reader = new StreamReader(new UnicodeReader(new FileInputStream( 96 getFileByName(dataName)))); 97 Scanner scanner = new ScannerImpl(reader); 98 try { 99 while (scanner.checkToken(new Token.ID[0])) { 100 Token token = scanner.getToken(); 101 if (!(token instanceof StreamStartToken || token instanceof StreamEndToken)) { 102 String replacement = replaces.get(token.getClass()); 103 tokens1.add(replacement); 104 } 105 } 106 assertEquals(tokenFileData, tokens1.size(), tokens2.size()); 107 assertEquals(tokens1, tokens2); 108 } catch (RuntimeException e) { 109 System.out.println("File name: \n" + tokensFiles[i].getName()); 110 String data = getResource(tokensFiles[i].getName()); 111 System.out.println("Data: \n" + data); 112 System.out.println("Tokens:"); 113 for (String token : tokens1) { 114 System.out.println(token); 115 } 116 fail("Cannot scan: " + tokensFiles[i]); 117 } 118 } 119 } 120 121 public void testScanner() throws IOException { 122 File[] files = getStreamsByExtension(".data", true); 123 assertTrue("No test files found.", files.length > 0); 124 for (File file : files) { 125 List<String> tokens = new ArrayList<String>(); 126 InputStream input = new FileInputStream(file); 127 StreamReader reader = new StreamReader(new UnicodeReader(input)); 128 Scanner scanner = new ScannerImpl(reader); 129 try { 130 while (scanner.checkToken(new Token.ID[0])) { 131 Token token = scanner.getToken(); 132 tokens.add(token.getClass().getName()); 133 } 134 } catch (RuntimeException e) { 135 System.out.println("File name: \n" + file.getName()); 136 String data = getResource(file.getName()); 137 System.out.println("Data: \n" + data); 138 System.out.println("Tokens:"); 139 for (String token : tokens) { 140 System.out.println(token); 141 } 142 fail("Cannot scan: " + file + "; " + e.getMessage()); 143 } finally { 144 input.close(); 145 } 146 } 147 } 148 } 149
