Skip to content

Commit cf0daec

Browse files
steve-sansalond
steve-s
authored and
ansalond
committed
[GR-21590] [GR-65456] Update imports, implement new FileSystem methods.
PullRequest: graalpython/3824
2 parents 68be29c + e5d33de commit cf0daec

File tree

12 files changed

+164
-66
lines changed

12 files changed

+164
-66
lines changed

‎ci.jsonnet‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{ "overlay": "d1b07bcbc3aec162c1a63abc3b81a574daae3963" }
1+
{ "overlay": "085931c8fed2b4635554b514af888554ef775385" }

‎graalpython/com.oracle.graal.python.test.integration/src/GRAALPY-VFS/com.oracle.graal.python.test/integration/fileslist.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
/GRAALPY-VFS/com.oracle.graal.python.test/integration/emptydir/
66
/GRAALPY-VFS/com.oracle.graal.python.test/integration/file1
77
/GRAALPY-VFS/com.oracle.graal.python.test/integration/extractme
8-
/GRAALPY-VFS/com.oracle.graal.python.test/integration/fileslist.txt
98
/GRAALPY-VFS/com.oracle.graal.python.test/integration/SomeFile
109
/GRAALPY-VFS/com.oracle.graal.python.test/integration/site-packages/
1110
/GRAALPY-VFS/com.oracle.graal.python.test/integration/site-packages/testpkg/

‎graalpython/com.oracle.graal.python.test.integration/src/org.graalvm.python.vfs/fileslist.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
/org.graalvm.python.vfs/emptydir/
66
/org.graalvm.python.vfs/file1
77
/org.graalvm.python.vfs/extractme
8-
/org.graalvm.python.vfs/fileslist.txt
98
/org.graalvm.python.vfs/SomeFile
109
/org.graalvm.python.vfs/site-packages/
1110
/org.graalvm.python.vfs/site-packages/testpkg/

‎graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/test/integration/VirtualFileSystemIntegrationTest.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -441,12 +441,11 @@ public void fsOperations(Context ctx, String pathPrefix) {
441441
assert len(f) == 0, 'expected no files'
442442
443443
f = listdir('/test_mount_point/')
444-
assert len(f) == 7, 'expected 7 files, got ' + str(len(f))
444+
assert len(f) == 6, 'expected 6 files, got ' + repr(list(f))
445445
446446
assert 'dir1' in f, 'does not contain "dir1"'
447447
assert 'emptydir' in f, 'does not contain "emptydir"'
448448
assert 'file1' in f, 'does not contain "file1"'
449-
assert 'fileslist.txt' in f, 'does not contain "fileslist.txt"'
450449
451450
f = listdir('{pathPrefix}dir1')
452451
if len(f) != 2:
@@ -485,7 +484,7 @@ assert len(f) == 0, 'expected no files in emptydir'
485484
dirs.add(r + "/" + dd)
486485
487486
assert len(roots) == 9, 'expected 10 roots, got ' + str(len(roots))
488-
assert len(files) == 15, 'expected 15 files, got ' + str(len(files))
487+
assert len(files) == 14, 'expected 14 files, got ' + str(len(files))
489488
assert len(dirs) == 8, 'expected 8 dirs, got ' + str(len(dirs))
490489
""", pathPrefix);
491490

‎graalpython/com.oracle.graal.python.test/src/org/graalvm/python/embedding/test/VirtualFileSystemTest.java‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,15 @@
4141

4242
package org.graalvm.python.embedding.test;
4343

44-
import org.graalvm.polyglot.io.FileSystem;
45-
import org.graalvm.python.embedding.VirtualFileSystem;
46-
import org.junit.After;
47-
import org.junit.Assert;
48-
import org.junit.Before;
49-
import org.junit.Test;
44+
import static com.oracle.graal.python.test.integration.Utils.IS_WINDOWS;
45+
import static org.graalvm.python.embedding.VirtualFileSystem.HostIO.NONE;
46+
import static org.graalvm.python.embedding.VirtualFileSystem.HostIO.READ;
47+
import static org.graalvm.python.embedding.VirtualFileSystem.HostIO.READ_WRITE;
48+
import static org.junit.Assert.assertEquals;
49+
import static org.junit.Assert.assertFalse;
50+
import static org.junit.Assert.assertNotEquals;
51+
import static org.junit.Assert.assertTrue;
52+
import static org.junit.Assert.fail;
5053

5154
import java.io.File;
5255
import java.io.IOException;
@@ -82,15 +85,12 @@
8285
import java.util.logging.Logger;
8386
import java.util.stream.Stream;
8487

85-
import static com.oracle.graal.python.test.integration.Utils.IS_WINDOWS;
86-
import static org.graalvm.python.embedding.VirtualFileSystem.HostIO.NONE;
87-
import static org.graalvm.python.embedding.VirtualFileSystem.HostIO.READ;
88-
import static org.graalvm.python.embedding.VirtualFileSystem.HostIO.READ_WRITE;
89-
import static org.junit.Assert.assertEquals;
90-
import static org.junit.Assert.assertFalse;
91-
import static org.junit.Assert.assertNotEquals;
92-
import static org.junit.Assert.assertTrue;
93-
import static org.junit.Assert.fail;
88+
import org.graalvm.polyglot.io.FileSystem;
89+
import org.graalvm.python.embedding.VirtualFileSystem;
90+
import org.junit.After;
91+
import org.junit.Assert;
92+
import org.junit.Before;
93+
import org.junit.Test;
9494

9595
public class VirtualFileSystemTest {
9696

@@ -740,7 +740,7 @@ public void noExtractFilter() throws Exception {
740740
extractFilter(null).//
741741
resourceLoadingClass(VirtualFileSystemTest.class).build()) {
742742
FileSystem fs = getDelegatingFS(vfs);
743-
assertEquals(23, checkNotExtracted(fs, VFS_ROOT_PATH));
743+
assertEquals(22, checkNotExtracted(fs, VFS_ROOT_PATH));
744744
}
745745
}
746746

@@ -1130,6 +1130,29 @@ public void copy() throws Exception {
11301130

11311131
}
11321132

1133+
@Test
1134+
public void testIsReadOnly() throws Exception {
1135+
for (FileSystem fs : new FileSystem[]{rwHostIOVFS, rHostIOVFS, noHostIOVFS}) {
1136+
assertTrue(fs.isFileStoreReadOnly(VFS_ROOT_PATH.resolve("file1")));
1137+
Assert.assertThrows(NoSuchFileException.class, () -> fs.isFileStoreReadOnly(VFS_ROOT_PATH.resolve("bogus")));
1138+
}
1139+
}
1140+
1141+
@Test
1142+
public void testGetFileStoreSpace() throws Exception {
1143+
for (FileSystem fs : new FileSystem[]{rwHostIOVFS, rHostIOVFS, noHostIOVFS}) {
1144+
long expectedSize = IS_WINDOWS ? 1162 : 1158; // this is because of different newlines
1145+
assertEquals(expectedSize, fs.getFileStoreTotalSpace(VFS_ROOT_PATH.resolve("file1")));
1146+
assertEquals(0, fs.getFileStoreUnallocatedSpace(VFS_ROOT_PATH.resolve("file1")));
1147+
assertEquals(0, fs.getFileStoreUsableSpace(VFS_ROOT_PATH.resolve("file1")));
1148+
assertEquals(4096, fs.getFileStoreBlockSize(VFS_ROOT_PATH.resolve("file1")));
1149+
Assert.assertThrows(NoSuchFileException.class, () -> fs.isFileStoreReadOnly(VFS_ROOT_PATH.resolve("bogus")));
1150+
Assert.assertThrows(NoSuchFileException.class, () -> fs.getFileStoreUnallocatedSpace(VFS_ROOT_PATH.resolve("bogus")));
1151+
Assert.assertThrows(NoSuchFileException.class, () -> fs.getFileStoreUsableSpace(VFS_ROOT_PATH.resolve("bogus")));
1152+
Assert.assertThrows(NoSuchFileException.class, () -> fs.getFileStoreBlockSize(VFS_ROOT_PATH.resolve("bogus")));
1153+
}
1154+
}
1155+
11331156
@Test
11341157
public void testImpl() throws Exception {
11351158
Set<String> ignored = Set.of(

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_tarfile.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ test.test_tarfile.WriteTest.test_fileobj_no_close @ darwin-arm64,darwin-x86_64,l
550550
test.test_tarfile.WriteTest.test_filter @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
551551
test.test_tarfile.WriteTest.test_gettarinfo_pathlike_name @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
552552
test.test_tarfile.WriteTest.test_link_size @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
553-
test.test_tarfile.WriteTest.test_open_nonwritable_fileobj @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
553+
test.test_tarfile.WriteTest.test_open_nonwritable_fileobj @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
554554
test.test_tarfile.WriteTest.test_ordered_recursion @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
555555
test.test_tarfile.WriteTest.test_pathnames @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
556556
test.test_tarfile.WriteTest.test_symlink_size @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_zipfile.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ test.test_zipfile.TestPath.test_open_extant_directory @ darwin-arm64,darwin-x86_
246246
test.test_zipfile.TestPath.test_open_missing_directory @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
247247
test.test_zipfile.TestPath.test_open_write @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
248248
test.test_zipfile.TestPath.test_parent @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
249-
test.test_zipfile.TestPath.test_pathlike_construction @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
249+
test.test_zipfile.TestPath.test_pathlike_construction @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
250250
test.test_zipfile.TestPath.test_read @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
251251
test.test_zipfile.TestPath.test_read_does_not_close @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
252252
test.test_zipfile.TestPath.test_root_name @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64

‎graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/GraalPyResources.java‎

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@
126126
* }
127127
* }
128128
* </pre>
129-
*
129+
*
130130
* In this example we:
131131
* <ul>
132132
* <li>create a {@link VirtualFileSystem} configured to have the root
@@ -152,7 +152,7 @@
152152
* <p>
153153
* <b>Example</b> creating a GraalPy context configured for the usage with an external resource
154154
* directory:
155-
*
155+
*
156156
* <pre>
157157
* try (Context context = GraalPyResources.contextBuilder(Path.of("python-resources")).build()) {
158158
* context.eval("python", "import mymodule; mymodule.print_hello_world()");
@@ -164,7 +164,7 @@
164164
* }
165165
* }
166166
* </pre>
167-
*
167+
*
168168
* In this example we:
169169
* <ul>
170170
* <li>create a GraalPy context which is preconfigured with GraalPy resources in an external
@@ -191,8 +191,9 @@ private GraalPyResources() {
191191

192192
/**
193193
* Creates a GraalPy context preconfigured with a {@link VirtualFileSystem} and other GraalPy
194-
* and polyglot Context configuration options optimized for the usage of the Python virtual
195-
* environment contained in the virtual filesystem.
194+
* and polyglot Context configuration options optimized for the usage of the
195+
* <a href="https://docs.python.org/3/library/venv.html">Python virtual environment</a>
196+
* contained in the virtual filesystem.
196197
* <p>
197198
* Following resource paths are preconfigured:
198199
* <ul>
@@ -216,8 +217,9 @@ public static Context createContext() {
216217

217218
/**
218219
* Creates a GraalPy context builder preconfigured with a {@link VirtualFileSystem} and other
219-
* GraalPy and polyglot Context configuration options optimized for the usage of the Python
220-
* virtual environment contained in the virtual filesystem.
220+
* GraalPy and polyglot Context configuration options optimized for the usage of the
221+
* <a href="https://docs.python.org/3/library/venv.html">Python virtual environment</a>
222+
* contained in the virtual filesystem.
221223
* <p>
222224
* Following resource paths are preconfigured:
223225
* <ul>
@@ -246,7 +248,7 @@ public static Context createContext() {
246248
* use {@link #contextBuilder(VirtualFileSystem)} and
247249
* {@link VirtualFileSystem.Builder#resourceDirectory(String)} when building the
248250
* {@link VirtualFileSystem}.
249-
*
251+
*
250252
* @see <a href=
251253
* "https://github.com/oracle/graalpython/blob/master/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java">PythonOptions</a>
252254
* @return a new {@link org.graalvm.polyglot.Context.Builder} instance
@@ -259,8 +261,9 @@ public static Context.Builder contextBuilder() {
259261

260262
/**
261263
* Creates a GraalPy context builder preconfigured with the given {@link VirtualFileSystem} and
262-
* other GraalPy and polygot Context configuration options optimized for the usage of the Python
263-
* virtual environment contained in the virtual filesystem.
264+
* other GraalPy and polygot Context configuration options optimized for the usage of the
265+
* <a href="https://docs.python.org/3/library/venv.html">Python virtual environment</a>
266+
* contained in the virtual filesystem.
264267
* <p>
265268
* Following resource paths are preconfigured:
266269
* <ul>
@@ -288,7 +291,7 @@ public static Context.Builder contextBuilder() {
288291
* }
289292
* }
290293
* </pre>
291-
*
294+
*
292295
* In this example we:
293296
* <ul>
294297
* <li>create a {@link VirtualFileSystem} configured to have the root
@@ -333,7 +336,7 @@ public static Context.Builder contextBuilder(VirtualFileSystem vfs) {
333336
* </p>
334337
* <p>
335338
* <b>Example</b>
336-
*
339+
*
337340
* <pre>
338341
* Context.Builder builder = GraalPyResources.contextBuilder(Path.of("python-resources"));
339342
* try (Context context = builder.build()) {
@@ -346,17 +349,31 @@ public static Context.Builder contextBuilder(VirtualFileSystem vfs) {
346349
* }
347350
* }
348351
* </pre>
349-
*
352+
*
350353
* In this example we:
351354
* <ul>
352355
* <li>create a GraalPy context which is preconfigured with GraalPy resources in an external
353356
* resource directory</li>
354357
* <li>use the context to import the python module <code>mymodule</code>, which should be either
355358
* located in <code>python-resources/src</code> or in a python package installed in
356359
* <code>/python/venv</code> (python virtual environment)</li>
360+
* <li>note that in this scenario, the Python context has access to the extracted resources as
361+
* well as the rest of the real filesystem</li>
357362
* </ul>
358363
* </p>
359364
*
365+
* <p>
366+
* External resources directory is often used for better compatibility with Python native
367+
* extensions that may bypass the Python abstractions and access the filesystem directly from
368+
* native code. Setting the {@code PosixModuleBackend} option to "native" increases the
369+
* compatibility further, but in such case even Python code bypasses the Truffle abstractions
370+
* and accesses native POSIX APIs directly. Usage:
371+
*
372+
* <pre>
373+
* GraalPyResources.contextBuilder(Path.of("python-resources")).option("python.PosixModuleBackend", "native")
374+
* </pre>
375+
* </p>
376+
*
360377
* <p/>
361378
* When Maven or Gradle GraalPy plugin is used to build the virtual environment, it also has to
362379
* be configured to generate the virtual environment into the same directory using the

‎graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/VirtualFileSystem.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -229,9 +229,9 @@ public Builder resourceLoadingClass(Class<?> c) {
229229
/**
230230
* This filter applied to files in the virtual filesystem treats them as symlinks to real
231231
* files in the host filesystem. This is useful, for example, if files in the virtual
232-
* filesystem need to be accessed outside the Truffle sandbox. They will be extracted to the
233-
* Java temporary directory on demand. The default filter matches any DLLs, dynamic
234-
* libraries, shared objects, and Python C extension files, because these need to be
232+
* filesystem need to be accessed outside the Truffle IO virtualization. They will be
233+
* extracted to the Java temporary directory on demand. The default filter matches any DLLs,
234+
* dynamic libraries, shared objects, and Python C extension files, because these need to be
235235
* accessed by the operating system loader. Setting this filter to <code>null</code> denies
236236
* any extraction. Any other filter is combined with the default filter.
237237
*

0 commit comments

Comments
 (0)