From e94efe88cea3f86cacd5e138455a23dc36fcc9bd Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 8 Sep 2026 12:06:21 +0200 Subject: [PATCH] Implement msvcrt.locking properly --- .../src/tests/unittest_tags/test_msvcrt.txt | 2 ++ .../modules/MsvcrtModuleBuiltins.java | 21 +++++++---- .../python/runtime/EmulatedPosixSupport.java | 36 +++++++++++-------- .../python/runtime/LoggingPosixSupport.java | 11 ++++++ .../python/runtime/NativePosixSupport.java | 10 ++++++ .../graal/python/runtime/PosixConstants.java | 12 +++++++ .../python/runtime/PosixConstantsWin32.java | 12 +++++-- .../python/runtime/PosixSupportLibrary.java | 2 ++ .../python/runtime/PreInitPosixSupport.java | 7 ++++ graalpython/python-libposix/src/posix.c | 20 +++++++++++ scripts/gen_native_cfg.py | 21 +++++++++-- 11 files changed, 128 insertions(+), 26 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_msvcrt.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_msvcrt.txt index 1ca91f3277..aeea7501f7 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_msvcrt.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_msvcrt.txt @@ -1,3 +1,5 @@ test.test_msvcrt.TestFileOperations.test_get_osfhandle @ win32-AMD64,win32-AMD64-github +test.test_msvcrt.TestFileOperations.test_locking @ win32-AMD64,win32-AMD64-github test.test_msvcrt.TestFileOperations.test_open_osfhandle @ win32-AMD64,win32-AMD64-github test.test_msvcrt.TestFileOperations.test_setmode @ win32-AMD64,win32-AMD64-github +test.test_msvcrt.TestFileOperations.test_unlockfile @ win32-AMD64,win32-AMD64-github diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java index 610135b411..69d21c7834 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java @@ -57,6 +57,8 @@ import com.oracle.graal.python.nodes.function.builtins.PythonTernaryClinicBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryClinicBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider; +import com.oracle.graal.python.runtime.GilNode; +import com.oracle.graal.python.runtime.PosixConstants; import com.oracle.graal.python.runtime.PosixSupportLibrary; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; @@ -71,9 +73,6 @@ @CoreFunctions(defineModule = "msvcrt", os = PythonOS.PLATFORM_WIN32) public final class MsvcrtModuleBuiltins extends PythonBuiltins { - public static final int LK_LOCK = 1; - public static final int LK_NBLCK = 2; - public static final int LK_UNLOCK = 3; private static final TruffleString T_MSVCRT_LOCKING = tsLiteral("msvcrt.locking"); private static final TruffleString T_MSVCRT_GET_OSFHANDLE = tsLiteral("msvcrt.get_osfhandle"); private static final TruffleString T_MSVCRT_OPEN_OSFHANDLE = tsLiteral("msvcrt.open_osfhandle"); @@ -86,9 +85,11 @@ protected List> getNodeFa @Override public void initialize(Python3Core core) { super.initialize(core); - addBuiltinConstant("LK_LOCK", LK_LOCK); - addBuiltinConstant("LK_NBLCK", LK_NBLCK); - addBuiltinConstant("LK_UNLCK", LK_UNLOCK); + addBuiltinConstant("LK_LOCK", PosixConstants._LK_LOCK.getValueIfDefined()); + addBuiltinConstant("LK_NBLCK", PosixConstants._LK_NBLCK.getValueIfDefined()); + addBuiltinConstant("LK_UNLCK", PosixConstants._LK_UNLCK.getValueIfDefined()); + addBuiltinConstant("LK_RLCK", PosixConstants._LK_RLCK.getValueIfDefined()); + addBuiltinConstant("LK_NBRLCK", PosixConstants._LK_NBRLCK.getValueIfDefined()); } @Builtin(name = "locking", minNumOfPositionalArgs = 3, parameterNames = {"fd", "mode", "nbytes"}) @@ -102,10 +103,16 @@ Object locking(VirtualFrame frame, int fd, int mode, long nbytes, @Bind Node inliningTarget, @Cached SysModuleBuiltins.AuditNode auditNode, @CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib, + @Cached GilNode gilNode, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { auditNode.audit(frame, inliningTarget, T_MSVCRT_LOCKING, fd, mode, nbytes); try { - posixLib.fcntlLock(getPosixSupport(), fd, mode != LK_NBLCK, mode == LK_UNLOCK ? 0 : 1, 0, 0, nbytes); + gilNode.release(true); + try { + posixLib.msvcrtLocking(getPosixSupport(), fd, mode, nbytes); + } finally { + gilNode.acquire(); + } } catch (PosixSupportLibrary.PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java index 5b708438ec..a766102861 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java @@ -613,6 +613,25 @@ public int setMode(int fd, int mode) throws PosixException { return previousMode; } + @ExportMessage + public void msvcrtLocking(int fd, int mode, long nbytes, + @Bind Node inliningTarget, + @Shared("errorBranch") @Cached InlinedBranchProfile errorBranch) throws PosixException { + Channel channel = getFileChannel(fd); + if (channel == null) { + errorBranch.enter(inliningTarget); + throw posixException(OSErrorEnum.EBADF); + } + boolean unlock = mode == PosixConstants._LK_UNLCK.getValueIfDefined(); + boolean blocking = mode == PosixConstants._LK_LOCK.getValueIfDefined() || mode == PosixConstants._LK_RLCK.getValueIfDefined(); + boolean nonBlocking = mode == PosixConstants._LK_NBLCK.getValueIfDefined() || mode == PosixConstants._LK_NBRLCK.getValueIfDefined(); + if (!unlock && !blocking && !nonBlocking) { + errorBranch.enter(inliningTarget); + throw posixException(OSErrorEnum.EINVAL); + } + doLockOperation(fd, channel, unlock, false, blocking, SEEK_CUR.value, 0, nbytes); + } + @ExportMessage(name = "pipe") public int[] pipeMessage(@Shared("eq") @Cached TruffleString.EqualNode eqNode) throws PosixException { // TODO: will merge with super.pipe once the super class is merged with this class @@ -908,20 +927,9 @@ void fcntlLock(int fd, boolean blocking, int lockType, int whence, long start, l errorBranch.enter(inliningTarget); throw posixException(OSErrorEnum.EBADFD); } - boolean unlock, shared, exclusive; - if (PythonLanguage.getPythonOS() == PLATFORM_WIN32) { - /* - * Windows doesn't expose fnctl, but we call this from MsvcrtModuleBuiltins, where we - * use 0 for unlock and 1 for lock - */ - unlock = lockType == 0; - exclusive = !unlock; - shared = false; - } else { - unlock = lockType == F_UNLCK.getValueIfDefined(); - shared = lockType == F_RDLCK.getValueIfDefined(); - exclusive = lockType == F_WRLCK.getValueIfDefined(); - } + boolean unlock = lockType == F_UNLCK.getValueIfDefined(); + boolean shared = lockType == F_RDLCK.getValueIfDefined(); + boolean exclusive = lockType == F_WRLCK.getValueIfDefined(); if (!unlock && !shared && !exclusive) { errorBranch.enter(inliningTarget); throw posixException(OSErrorEnum.EINVAL); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java index 5675cef732..de40c75409 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java @@ -274,6 +274,17 @@ final int setMode(int fd, int mode, } } + @ExportMessage + final void msvcrtLocking(int fd, int mode, long nbytes, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws PosixException { + logEnter("msvcrtLocking", "%d, %d, %d", fd, mode, nbytes); + try { + lib.msvcrtLocking(delegate, fd, mode, nbytes); + } catch (PosixException e) { + throw logException("msvcrtLocking", e); + } + } + @ExportMessage final int[] pipe( @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws PosixException { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java index a8ad6a182b..e1ffd6e3ad 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java @@ -241,6 +241,9 @@ abstract static class PosixNativeFunctionInvoker { @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32, SINT32}) abstract int call_setmode(int fd, int mode); + @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32, SINT32, SINT64}) + abstract int call_msvcrt_locking(int fd, int mode, long nbytes); + @DowncallSignature(returnType = SINT32, argumentTypes = {POINTER}) abstract int call_pipe2(long pipefd); @@ -849,6 +852,13 @@ public int setMode(int fd, int mode) throws PosixException { return previousMode; } + @ExportMessage + public void msvcrtLocking(int fd, int mode, long nbytes) throws PosixException { + if (posixNativeFunctionInvoker.call_msvcrt_locking(fd, mode, nbytes) != 0) { + throw getErrnoAndThrowPosixException(); + } + } + @ExportMessage public int[] pipe() throws PosixException { int[] fds = new int[2]; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstants.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstants.java index 91a913a6d2..dd33f9b75d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstants.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstants.java @@ -165,6 +165,11 @@ public final class PosixConstants { public static final OptionalIntConstant F_RDLCK; public static final OptionalIntConstant F_WRLCK; public static final OptionalIntConstant F_UNLCK; + public static final OptionalIntConstant _LK_UNLCK; + public static final OptionalIntConstant _LK_LOCK; + public static final OptionalIntConstant _LK_NBLCK; + public static final OptionalIntConstant _LK_RLCK; + public static final OptionalIntConstant _LK_NBRLCK; public static final MandatoryIntConstant DT_UNKNOWN; public static final MandatoryIntConstant DT_FIFO; public static final MandatoryIntConstant DT_CHR; @@ -388,6 +393,7 @@ public final class PosixConstants { public static final IntConstant[] mmapProtection; public static final IntConstant[] flockOperation; public static final IntConstant[] flockType; + public static final IntConstant[] msvcrtLocking; public static final IntConstant[] direntType; public static final IntConstant[] waitOptions; public static final IntConstant[] accessMode; @@ -498,6 +504,11 @@ public final class PosixConstants { F_RDLCK = reg.createOptionalInt("F_RDLCK"); F_WRLCK = reg.createOptionalInt("F_WRLCK"); F_UNLCK = reg.createOptionalInt("F_UNLCK"); + _LK_UNLCK = reg.createOptionalInt("_LK_UNLCK"); + _LK_LOCK = reg.createOptionalInt("_LK_LOCK"); + _LK_NBLCK = reg.createOptionalInt("_LK_NBLCK"); + _LK_RLCK = reg.createOptionalInt("_LK_RLCK"); + _LK_NBRLCK = reg.createOptionalInt("_LK_NBRLCK"); DT_UNKNOWN = reg.createMandatoryInt("DT_UNKNOWN"); DT_FIFO = reg.createMandatoryInt("DT_FIFO"); DT_CHR = reg.createMandatoryInt("DT_CHR"); @@ -723,6 +734,7 @@ public final class PosixConstants { mmapProtection = new IntConstant[]{PROT_NONE, PROT_READ, PROT_WRITE, PROT_EXEC}; flockOperation = new IntConstant[]{LOCK_SH, LOCK_EX, LOCK_NB, LOCK_UN}; flockType = new IntConstant[]{F_RDLCK, F_WRLCK, F_UNLCK}; + msvcrtLocking = new IntConstant[]{_LK_UNLCK, _LK_LOCK, _LK_NBLCK, _LK_RLCK, _LK_NBRLCK}; direntType = new IntConstant[]{DT_UNKNOWN, DT_FIFO, DT_CHR, DT_DIR, DT_BLK, DT_REG, DT_LNK, DT_SOCK, DT_WHT}; waitOptions = new IntConstant[]{WNOHANG, WUNTRACED}; accessMode = new IntConstant[]{R_OK, W_OK, X_OK, F_OK}; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java index 7233591050..841b81aa00 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java @@ -40,8 +40,8 @@ */ package com.oracle.graal.python.runtime; -// Auto generated by gen_native_cfg.py at 2025-08-19 12:47:42.927963 -// on Windows QUICKEM-GVMHCD3 11 10.0.22631 AMD64 Intel64 Family 6 Model 154 Stepping 3, GenuineIntel +// Auto generated by gen_native_cfg.py at 2026-09-08 12:57:44.557226 +// on Windows QUICKEM-GVMHCD3 11 10.0.22631 AMD64 Intel64 Family 6 Model 198 Stepping 2, GenuineIntel class PosixConstantsWin32 { private PosixConstantsWin32() { @@ -76,6 +76,9 @@ static void getConstants(PosixConstants.Registry constants) { constants.put("O_TEMPORARY", 0x00000040); constants.put("O_BINARY", 0x00008000); constants.put("O_TEXT", 0x00004000); + constants.put("O_NOINHERIT", 0x00000080); + constants.put("O_RANDOM", 0x00000010); + constants.put("O_SEQUENTIAL", 0x00000020); constants.put("S_IFMT", 0x0000F000); constants.put("S_IFSOCK", 0); constants.put("S_IFLNK", 0); @@ -94,6 +97,11 @@ static void getConstants(PosixConstants.Registry constants) { constants.put("LOCK_EX", 0); constants.put("LOCK_NB", 0); constants.put("LOCK_UN", 0); + constants.put("_LK_UNLCK", 0); + constants.put("_LK_LOCK", 1); + constants.put("_LK_NBLCK", 2); + constants.put("_LK_RLCK", 3); + constants.put("_LK_NBRLCK", 4); constants.put("DT_UNKNOWN", 0); constants.put("DT_FIFO", 0); constants.put("DT_CHR", 0); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java index 6e1a56b8c8..552bec4bc0 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java @@ -110,6 +110,8 @@ public abstract class PosixSupportLibrary extends Library { public abstract int setMode(Object receiver, int fd, int mode) throws PosixException; + public abstract void msvcrtLocking(Object receiver, int fd, int mode, long nbytes) throws PosixException; + public abstract int[] pipe(Object receiver) throws PosixException; public abstract SelectResult select(Object receiver, int[] readfds, int[] writefds, int[] errorfds, Timeval timeout) throws PosixException; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java index 35fba5be23..281004c367 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java @@ -263,6 +263,13 @@ final int setMode(int fd, int mode, return nativeLib.setMode(nativePosixSupport, fd, mode); } + @ExportMessage + final void msvcrtLocking(int fd, int mode, long nbytes, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws PosixException { + checkNotInPreInitialization(); + nativeLib.msvcrtLocking(nativePosixSupport, fd, mode, nbytes); + } + @ExportMessage final int[] pipe(@CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws PosixException { checkNotInPreInitialization(); diff --git a/graalpython/python-libposix/src/posix.c b/graalpython/python-libposix/src/posix.c index d6a21f3479..d7e0b2c374 100644 --- a/graalpython/python-libposix/src/posix.c +++ b/graalpython/python-libposix/src/posix.c @@ -1237,6 +1237,17 @@ GP_EXPORT int32_t call_setmode(int32_t fd, int32_t mode) { return previous_mode; } +GP_EXPORT int32_t call_msvcrt_locking(int32_t fd, int32_t mode, int64_t nbytes) { + int result; + BEGIN_SUPPRESS_IPH + result = _locking(fd, mode, (long) nbytes); + END_SUPPRESS_IPH + if (result != 0) { + capture_errno(); + } + return result; +} + GP_EXPORT int32_t call_pipe2(int32_t *pipefd) { int result = _pipe(pipefd, 8192, _O_BINARY | _O_NOINHERIT); if (result < 0) { @@ -2411,6 +2422,15 @@ int32_t call_setmode(int32_t fd, int32_t mode) { return -1; } +int32_t call_msvcrt_locking(int32_t fd, int32_t mode, int64_t nbytes) { + (void) fd; + (void) mode; + (void) nbytes; + errno = ENOSYS; + capture_errno(); + return -1; +} + int32_t call_pipe2(int32_t *pipefd) { #ifdef __gnu_linux__ CAPTURE_ERRNO_AND_RETURN(-1, pipe2(pipefd, O_CLOEXEC)); diff --git a/scripts/gen_native_cfg.py b/scripts/gen_native_cfg.py index 5f79ab65ee..7a7e1df262 100644 --- a/scripts/gen_native_cfg.py +++ b/scripts/gen_native_cfg.py @@ -88,6 +88,7 @@ # include # include # include +# include # include # ifndef PATH_MAX # define PATH_MAX MAX_PATH @@ -231,6 +232,13 @@ * i F_WRLCK * i F_UNLCK +[msvcrtLocking] +* i _LK_UNLCK +* i _LK_LOCK +* i _LK_NBLCK +* i _LK_RLCK +* i _LK_NBRLCK + [direntType] 0 i DT_UNKNOWN 0 i DT_FIFO @@ -514,7 +522,7 @@ ''' java_copyright = '''/* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -575,6 +583,7 @@ class PosixConstants{platform} {{ c_source_file = 'gen_native_cfg.c' c_executable_file = 'gen_native_cfg' +c_output_file = 'gen_native_cfg.out' def parse_defs(): @@ -678,7 +687,10 @@ def generate_platform(): cc = os.environ.get('CC', 'cl' if platform == 'Win32' else 'cc') subprocess.run(f'{cc} {flags} -o {c_executable_file} {c_source_file}', shell=True, check=True) - output = subprocess.run(f'./{c_executable_file}', shell=False, check=True, stdout=subprocess.PIPE, universal_newlines=True).stdout[:-1] + with open(c_output_file, 'w+') as output_file: + subprocess.run(f'./{c_executable_file}', shell=False, check=True, stdout=output_file, universal_newlines=True) + output_file.seek(0) + output = output_file.read()[:-1] uname = " ".join(tuple(plat.uname())) out_path = DIR / f'graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstants{platform}.java' @@ -762,7 +774,7 @@ def add_constant(opt, typ, name): def generate_native_constants(layouts): c_filename = DIR / 'graalpython/python-libposix/src/posix.c' - java_filename = DIR / 'graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIPosixConstants.java' + java_filename = DIR / 'graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixConstants.java' constants = [] for struct in layouts: if struct.unix_only: @@ -795,6 +807,9 @@ def main(): finally: delete_if_exists(c_source_file) delete_if_exists(c_executable_file) + delete_if_exists(c_executable_file + '.exe') + delete_if_exists(c_executable_file + '.obj') + delete_if_exists(c_output_file) if __name__ == '__main__':