feat: add full Unicode support for Javascript identifiers

.gitignore

@@ -12,3 +12,12 @@ target/
 /yarn.lock
 /node_modules
 /.metadata/
+
+# Bazel stuff
+/bazel-bin
+/bazel-closure-compiler
+/bazel-out
+/bazel-testlogs
+
+# IDE
+.vscode

src/com/google/javascript/jscomp/parsing/parser/Scanner.java

@@ -22,6 +22,8 @@
 import com.google.javascript.jscomp.parsing.parser.util.ErrorReporter;
 import com.google.javascript.jscomp.parsing.parser.util.SourcePosition;
 import com.google.javascript.jscomp.parsing.parser.util.SourceRange;
+import com.google.javascript.jscomp.parsing.parser.util.UnicodeMatch;
+
 import java.util.ArrayList;
 import javax.annotation.Nullable;
 
@@ -140,7 +142,7 @@ public LiteralToken nextRegularExpressionLiteralToken() {
     nextChar();
 
     // flags
-    while (isIdentifierPart(peekChar())) {
+    while (UnicodeMatch.isJavascriptIdentifierPart(peekChar())) {
       nextChar();
     }
 
@@ -764,7 +766,7 @@ private Token scanIdentifierOrKeyword(int beginToken, char ch) {
     int unicodeEscapeLen = containsUnicodeEscape ? 1 : 0;
 
     ch = peekChar();
-    while (isIdentifierPart(ch)
+    while (UnicodeMatch.isJavascriptIdentifierPart(ch)
         || ch == '\\'
         || (ch == '{' && unicodeEscapeLen == 2)
         || (ch == '}' && bracedUnicodeEscape)) {
@@ -804,7 +806,7 @@ private Token scanIdentifierOrKeyword(int beginToken, char ch) {
     // Check to make sure the first character (or the unicode escape at the
     // beginning of the identifier) is a valid identifier start character.
     char start = value.charAt(0);
-    if (!isIdentifierStart(start)) {
+    if (!UnicodeMatch.isJavascriptIdentifierStart(start)) {
       reportError(
           getPosition(beginToken),
           "Character '%c' (U+%04X) is not a valid identifier start char",
@@ -858,7 +860,7 @@ private static String processUnicodeEscapes(String value) {
         }
         // TODO(mattloring): Allow code points >= 0xFFFF (greater than the size of a char).
         char ch = (char) Integer.parseInt(hexDigits, 0x10);
-        if (!isIdentifierPart(ch)) {
+        if (!UnicodeMatch.isJavascriptIdentifierPart(ch)) {
           return null;
         }
         value = value.substring(0, escapeStart) + ch + value.substring(escapeEnd);
@@ -869,54 +871,6 @@ private static String processUnicodeEscapes(String value) {
     return value;
   }
 
-  @SuppressWarnings("ShortCircuitBoolean") // Intentional to minimize branches in this code
-  private static boolean isIdentifierStart(char ch) {
-    // Most code is written in pure ASCII, so create a fast path here.
-    if (ch <= 127) {
-      // Intentionally avoiding short circuiting behavior of "||" and "&&".
-      // This minimizes branches in this code which minimizes branch prediction misses.
-      return ((ch >= 'A' & ch <= 'Z') | (ch >= 'a' & ch <= 'z') | (ch == '_' | ch == '$'));
-    }
-
-    // Handle non-ASCII characters.
-    // TODO(tjgq): This should include all characters with the ID_Start property.
-    if (Character.isLetter(ch)) {
-      return true;
-    }
-
-    // Workaround for b/36459436.
-    // When running under GWT/J2CL, Character.isLetter only handles ASCII.
-    // Angular relies heavily on Latin Small Letter Barred O and Greek Capital Letter Delta.
-    // Greek letters are occasionally found in math code.
-    // Latin letters are found in our own tests.
-    return (ch >= 0x00C0 & ch <= 0x00D6) // Latin letters
-        // 0x00D7 = multiplication sign, not a letter
-        | (ch >= 0x00D8 & ch <= 0x00F6) // Latin letters
-        // 0x00F7 = division sign, not a letter
-        | (ch >= 0x00F8 & ch <= 0x00FF) // Latin letters
-        | ch == 0x0275 // Latin Barred O
-        | (ch >= 0x0391 & ch <= 0x03A1) // Greek uppercase letters
-        // 0x03A2 = unassigned
-        | (ch >= 0x03A3 & ch <= 0x03A9) // Remaining Greek uppercase letters
-        | (ch >= 0x03B1 & ch <= 0x03C9); // Greek lowercase letters
-  }
-
-  @SuppressWarnings("ShortCircuitBoolean") // Intentional to minimize branches in this code
-  private static boolean isIdentifierPart(char ch) {
-    // Most code is written in pure ASCII, so create a fast path here.
-    if (ch <= 127) {
-      return ((ch >= 'A' & ch <= 'Z')
-          | (ch >= 'a' & ch <= 'z')
-          | (ch >= '0' & ch <= '9')
-          | (ch == '_' | ch == '$'));
-    }
-
-    // Handle non-ASCII characters.
-    // TODO(tjgq): This should include all characters with the ID_Continue property, plus
-    // Zero Width Non-Joiner and Zero Width Joiner.
-    return isIdentifierStart(ch) || Character.isDigit(ch);
-  }
-
   private Token scanStringLiteral(int beginIndex, char terminator) {
     // String literals might span multiple lines.
     SourcePosition startingPosition = getPosition(beginIndex);

test/com/google/javascript/jscomp/deps/ClosureBundlerTest.java

@@ -210,33 +210,14 @@ public void testCommentsAndFormattingRemovedWithTranspilation() throws Exception
   }
 
   @Test
-  public void testFullWidthLowLineWithDefaultTranspilerIsOkay() throws Exception {
-    // The last character is something the compiler doesn't handle correctly
+  public void testFullWidthLowLine() throws Exception {
     String input = "var ａｅｓｔｈｅｔｉｃ＿";
     ClosureBundler bundler = new ClosureBundler();
     StringBuilder sb = new StringBuilder();
     bundler.appendTo(
         sb,
         SimpleDependencyInfo.builder("", "").build(),
         input);
-    assertThat(sb.toString()).isEqualTo(input);
-  }
-
-  // TODO(johnplaisted): If / when the compiler can parse full width low line in identifiers
-  // this should be okay to be transpiled.
-  @Test
-  public void testFullWidthLowLineInTranspiledCodeIsError() throws Exception {
-    // The last character is something the compiler doesn't handle correctly
-    String input = "let ａｅｓｔｈｅｔｉｃ＿";
-    ClosureBundler bundler = new ClosureBundler(BaseTranspiler.ES5_TRANSPILER);
-    StringBuilder sb = new StringBuilder();
-    try {
-      bundler.appendTo(sb, SimpleDependencyInfo.builder("", "").build(), input);
-      assertWithMessage("Expected an exception").fail();
-    } catch (TranspilationException e) {
-      assertThat(e)
-          .hasMessageThat()
-          .contains("Parse error. Character '＿' (U+FF3F) is not a valid identifier start char");
-    }
+    assertThat(sb.toString()).isEqualTo("var \uff41\uff45\uff53\uff54\uff48\uff45\uff54\uff49\uff43\uff3f");
   }
-}
+}

travis_util/test_npm.sh

@@ -19,6 +19,10 @@ if [ -z $TRAVIS_BUILD_DIR ]; then
   TRAVIS_BUILD_DIR=`pwd`
 fi
 
+if [ -z $TRAVIS_BUILD_DIR ]; then
+  TRAVIS_BUILD_DIR=`pwd`
+fi
+
 # Run yarn install so that dev dependencies are available
 yarn install && cd ${TRAVIS_BUILD_DIR}/node_modules/closure-compiler-npm && yarn install