Better caching

de.peeeq.wurstscript/.gitattributes

@@ -0,0 +1,4 @@
+* text=auto
+*.sh text eol=lf
+gradlew text eol=lf
+*.bat text eol=crlf

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/WurstChecker.java

@@ -39,12 +39,6 @@ public void checkProg(WurstModel root, Collection<CompilationUnit> toCheck) {
 
         if (errorHandler.getErrorCount() > 0) return;
 
-        // compute the flow attributes
-        for (CompilationUnit cu : toCheck) {
-            WurstValidator.computeFlowAttributes(cu);
-        }
-
-
         // validate the resource:
         WurstValidator validator = new WurstValidator(root);
         validator.validate(toCheck);

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/validation/GlobalCaches.java

@@ -1,12 +1,15 @@
 package de.peeeq.wurstscript.validation;
 
+import de.peeeq.wurstscript.ast.CompilationUnit;
 import de.peeeq.wurstscript.ast.Element;
+import de.peeeq.wurstscript.ast.WurstModel;
+import de.peeeq.wurstscript.attributes.AttrNearest;
 import de.peeeq.wurstscript.intermediatelang.ILconst;
 import de.peeeq.wurstscript.intermediatelang.interpreter.LocalState;
+import de.peeeq.wurstscript.jassIm.ImFunction;
 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
 
-import java.util.Arrays;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicLong;
 
 // Expose static fields only if you already have them there; otherwise, just clear via dedicated methods.
@@ -87,7 +90,7 @@ public boolean equals(Object o) {
 
     public enum Mode {TEST_ISOLATED, DEV_PERSISTENT}
 
-    private static volatile Mode mode = Mode.DEV_PERSISTENT;
+    public static volatile Mode mode = Mode.DEV_PERSISTENT;
 
     public static void setMode(Mode m) {
         mode = m;
@@ -130,6 +133,118 @@ public static void clearAll() {
         lookupCache.clear();
     }
 
+    /**
+     * Evict cache entries that are tied to any of the given compilation units.
+     *
+     * <p>The validator replaces only a subset of units on incremental runs. Instead of
+     * purging the full global caches (which would force re-computation for every file), we
+     * walk both caches and drop entries whose owner element/function belongs to one of the
+     * affected compilation units.</p>
+     */
+    public static void invalidateFor(WurstModel model, Collection<CompilationUnit> changedUnits) {
+        if (mode == Mode.TEST_ISOLATED) {
+            clearAll();
+            return;
+        }
+
+        Set<CompilationUnit> affected = toIdentitySet(changedUnits);
+        Set<CompilationUnit> live = model == null ? null : toIdentitySet(model);
+
+        if (affected.isEmpty() && (live == null || live.isEmpty())) {
+            return;
+        }
+
+        invalidateLookupCache(affected, live);
+        invalidateLocalStateCache(affected, live);
+    }
+
+    private static Set<CompilationUnit> toIdentitySet(Iterable<CompilationUnit> units) {
+        Set<CompilationUnit> set = Collections.newSetFromMap(new IdentityHashMap<>());
+        if (units == null) {
+            return set;
+        }
+        for (CompilationUnit cu : units) {
+            if (cu != null) {
+                set.add(cu);
+            }
+        }
+        return set;
+    }
+
+    private static void invalidateLookupCache(Set<CompilationUnit> affected, Set<CompilationUnit> live) {
+        if (lookupCache.isEmpty()) {
+            return;
+        }
+
+        int evicted = 0;
+        Iterator<Map.Entry<CacheKey, Object>> it = lookupCache.entrySet().iterator();
+        while (it.hasNext()) {
+            Map.Entry<CacheKey, Object> entry = it.next();
+            Element element = entry.getKey().element;
+            if (element == null) {
+                continue;
+            }
+
+            CompilationUnit owner = AttrNearest.nearestCompilationUnit(element);
+            if (owner == null) {
+                continue;
+            }
+
+            boolean shouldEvict = affected.contains(owner)
+                || (live != null && !live.contains(owner));
+
+            if (shouldEvict) {
+                it.remove();
+                evicted++;
+            }
+        }
+
+        if (evicted > 0) {
+            lookupStats.recordEviction(evicted);
+        }
+    }
+
+    private static void invalidateLocalStateCache(Set<CompilationUnit> affected, Set<CompilationUnit> live) {
+        if (LOCAL_STATE_CACHE.isEmpty()) {
+            return;
+        }
+
+        int evicted = 0;
+        Iterator<Map.Entry<Object, Object2ObjectOpenHashMap<ArgumentKey, LocalState>>> it =
+            LOCAL_STATE_CACHE.entrySet().iterator();
+        while (it.hasNext()) {
+            Map.Entry<Object, Object2ObjectOpenHashMap<ArgumentKey, LocalState>> entry = it.next();
+            Object key = entry.getKey();
+            if (!(key instanceof ImFunction)) {
+                continue;
+            }
+
+            ImFunction function = (ImFunction) key;
+            Element trace = function.attrTrace();
+            if (trace == null) {
+                continue;
+            }
+
+            CompilationUnit owner = AttrNearest.nearestCompilationUnit(trace);
+            if (owner == null) {
+                continue;
+            }
+
+            boolean shouldEvict = affected.contains(owner)
+                || (live != null && !live.contains(owner));
+
+            if (shouldEvict) {
+                it.remove();
+                evicted++;
+            }
+        }
+
+        if (evicted > 0) {
+            localStateStats.recordEviction(evicted);
+        }
+    }
+
+
     public enum LookupType {
         FUNC, VAR, TYPE, PACKAGE, MEMBER_FUNC, MEMBER_VAR
     }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/validation/WurstValidator.java

@@ -64,7 +64,7 @@ public void validate(Collection<CompilationUnit> toCheck) {
             visitedFunctions = 0;
             heavyFunctions.clear();
             heavyBlocks.clear();
-            GlobalCaches.clearAll();
+            GlobalCaches.invalidateFor(prog, toCheck);
 
             lightValidation(toCheck);
 
@@ -601,7 +601,7 @@ private int distanceToOwner(ClassDef start, ClassDef owner) {
 
     private void visit(StmtExitwhen exitwhen) {
         Element parent = exitwhen.getParent();
-        while (!(parent instanceof FunctionDefinition)) {
+        while (parent != null && !(parent instanceof FunctionDefinition)) {
             if (parent instanceof StmtForEach) {
                 StmtForEach forEach = (StmtForEach) parent;
                 if (forEach.getIn().tryGetNameDef().attrIsVararg()) {

de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/BugTests.java

@@ -71,7 +71,7 @@ public void bug62_codearray() {
 
     @Test
     public void bug61_break() {
-        testAssertErrorsLines(false, "inside a loop",
+        testAssertErrorsLines(false, "not allowed outside of loop",
                 "package test",
                 "	init",
                 "		break",

de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/ModelManagerTests.java

@@ -34,8 +34,7 @@
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.*;
 
 public class ModelManagerTests {
 
@@ -265,6 +264,116 @@ private Map<WFile, String> keepErrorsInMap(ModelManagerImpl manager) {
         return results;
     }
 
+    private CacheFixture setupCacheFixture(String projectName) throws IOException {
+        GlobalCaches.clearAll();
+
+        File projectFolder = new File("./temp/" + projectName + "/");
+        File wurstFolder = new File(projectFolder, "wurst");
+        newCleanFolder(wurstFolder);
+
+        String packageA = string(
+            "package A",
+            "import B",
+            "public function a() returns int",
+            "    return b()"
+        );
+
+        String packageB = string(
+            "package B",
+            "import C",
+            "public function b() returns int",
+            "    return c()"
+        );
+
+        String packageC = string(
+            "package C",
+            "public function c() returns int",
+            "    return 1"
+        );
+
+        String packageD = string(
+            "package D",
+            "public function d() returns int",
+            "    return 2"
+        );
+
+        WFile fileA = WFile.create(new File(wurstFolder, "A.wurst"));
+        WFile fileB = WFile.create(new File(wurstFolder, "B.wurst"));
+        WFile fileC = WFile.create(new File(wurstFolder, "C.wurst"));
+        WFile fileD = WFile.create(new File(wurstFolder, "D.wurst"));
+        WFile fileWurst = WFile.create(new File(wurstFolder, "Wurst.wurst"));
+
+        writeFile(fileA, packageA);
+        writeFile(fileB, packageB);
+        writeFile(fileC, packageC);
+        writeFile(fileD, packageD);
+        writeFile(fileWurst, "package Wurst\n");
+
+        ModelManagerImpl manager = new ModelManagerImpl(projectFolder, new BufferManager());
+        Map<WFile, String> results = keepErrorsInMap(manager);
+        manager.buildProject();
+
+        CompilationUnit cuA = manager.getCompilationUnit(fileA);
+        CompilationUnit cuB = manager.getCompilationUnit(fileB);
+        CompilationUnit cuC = manager.getCompilationUnit(fileC);
+        CompilationUnit cuD = manager.getCompilationUnit(fileD);
+
+        assertNotNull(cuA);
+        assertNotNull(cuB);
+        assertNotNull(cuC);
+        assertNotNull(cuD);
+
+        WPackage pkgA = cuA.getPackages().get(0);
+        WPackage pkgB = cuB.getPackages().get(0);
+        WPackage pkgC = cuC.getPackages().get(0);
+        WPackage pkgD = cuD.getPackages().get(0);
+
+        GlobalCaches.CacheKey keyA = new GlobalCaches.CacheKey(pkgA, "markerA", GlobalCaches.LookupType.FUNC);
+        GlobalCaches.CacheKey keyB = new GlobalCaches.CacheKey(pkgB, "markerB", GlobalCaches.LookupType.FUNC);
+        GlobalCaches.CacheKey keyC = new GlobalCaches.CacheKey(pkgC, "markerC", GlobalCaches.LookupType.FUNC);
+        GlobalCaches.CacheKey keyD = new GlobalCaches.CacheKey(pkgD, "markerD", GlobalCaches.LookupType.FUNC);
+
+        GlobalCaches.lookupCache.put(keyA, Boolean.TRUE);
+        GlobalCaches.lookupCache.put(keyB, Boolean.TRUE);
+        GlobalCaches.lookupCache.put(keyC, Boolean.TRUE);
+        GlobalCaches.lookupCache.put(keyD, Boolean.TRUE);
+
+        assertTrue(GlobalCaches.lookupCache.containsKey(keyA));
+        assertTrue(GlobalCaches.lookupCache.containsKey(keyB));
+        assertTrue(GlobalCaches.lookupCache.containsKey(keyC));
+        assertTrue(GlobalCaches.lookupCache.containsKey(keyD));
+
+        return new CacheFixture(manager, results, fileA, fileB, fileC, fileD, keyA, keyB, keyC, keyD);
+    }
+
+    private static final class CacheFixture {
+        final ModelManagerImpl manager;
+        final Map<WFile, String> results;
+        final WFile fileA;
+        final WFile fileB;
+        final WFile fileC;
+        final WFile fileD;
+        final GlobalCaches.CacheKey keyA;
+        final GlobalCaches.CacheKey keyB;
+        final GlobalCaches.CacheKey keyC;
+        final GlobalCaches.CacheKey keyD;
+
+        CacheFixture(ModelManagerImpl manager, Map<WFile, String> results, WFile fileA, WFile fileB,
+                     WFile fileC, WFile fileD, GlobalCaches.CacheKey keyA, GlobalCaches.CacheKey keyB,
+                     GlobalCaches.CacheKey keyC, GlobalCaches.CacheKey keyD) {
+            this.manager = manager;
+            this.results = results;
+            this.fileA = fileA;
+            this.fileB = fileB;
+            this.fileC = fileC;
+            this.fileD = fileD;
+            this.keyA = keyA;
+            this.keyB = keyB;
+            this.keyC = keyC;
+            this.keyD = keyD;
+        }
+    }
+
     private void newCleanFolder(File f) throws IOException {
         FileUtils.deleteRecursively(f);
         Files.createDirectories(f.toPath());
@@ -355,6 +464,49 @@ public void visit(ClassDef c) {
 
     }
 
+    @Test
+    public void selectiveCacheInvalidationSkipsUnaffectedUnits() throws IOException {
+        CacheFixture fixture = setupCacheFixture("cacheInvalidationProject1");
+        fixture.results.clear();
+
+        String packageBUpdated = string(
+            "package B",
+            "import C",
+            "public function b() returns int",
+            "    return c() + 1"
+        );
+
+        ModelManager.Changes changes = fixture.manager.syncCompilationUnitContent(fixture.fileB, packageBUpdated);
+        fixture.manager.reconcile(changes);
+
+        assertEquals(fixture.results.keySet(), ImmutableSet.of(fixture.fileA, fixture.fileB));
+        assertFalse(GlobalCaches.lookupCache.containsKey(fixture.keyA));
+        assertFalse(GlobalCaches.lookupCache.containsKey(fixture.keyB));
+        assertTrue(GlobalCaches.lookupCache.containsKey(fixture.keyC));
+        assertTrue(GlobalCaches.lookupCache.containsKey(fixture.keyD));
+    }
+
+    @Test
+    public void selectiveCacheInvalidationCoversTransitiveDependents() throws IOException {
+        CacheFixture fixture = setupCacheFixture("cacheInvalidationProject2");
+        fixture.results.clear();
+
+        String packageCUpdated = string(
+            "package C",
+            "public function c() returns int",
+            "    return 2"
+        );
+
+        ModelManager.Changes changes = fixture.manager.syncCompilationUnitContent(fixture.fileC, packageCUpdated);
+        fixture.manager.reconcile(changes);
+
+        assertEquals(fixture.results.keySet(), ImmutableSet.of(fixture.fileA, fixture.fileB, fixture.fileC));
+        assertFalse(GlobalCaches.lookupCache.containsKey(fixture.keyA));
+        assertFalse(GlobalCaches.lookupCache.containsKey(fixture.keyB));
+        assertFalse(GlobalCaches.lookupCache.containsKey(fixture.keyC));
+        assertTrue(GlobalCaches.lookupCache.containsKey(fixture.keyD));
+    }
+
     @Test
     public void changeModuleAbstractMethod() throws IOException {
         File projectFolder = new File("./temp/testProject2/");