Shell and terminal improved

Documentation/ideas.md

@@ -27,7 +27,6 @@
 - Improve Setup: Show "Step done" screen
 - Make it more clear to end-users that an SD card is required to run Tactility
 - Warn if file operations are done from prohibited tasks (e.g. lvgl task)
-- Shell interpreter works recursive-descent algorithm, which requires a big stack size. Can we optimize this?
 - stopAppFromToolbar() in Tactility.cpp stops the top-most app. Change it so the toolbar knows for which app id it is created, so it can rely on that.
 - Consider not unpacking `.app` files and executing them directly. Might want to cache file offsets. Cache file must be pinned to app version.
 - lvgl-module's spinner relies on hard-coded spinner asset from Tactility main project.

Modules/app-module/source/stdio_wrap.cpp

@@ -333,9 +333,9 @@ int chdir(const char* path) {
 // wraps above can't reach, so these redirect calls to printf/fprintf/etc directly. POSIX-only:
 // newlib's stdio already goes through the wrappable syscall stubs.
 //
-// Scoped to printf/getc: fread/fwrite are used sitewide for real file I/O, so wrapping them would
-// be a correctness risk for unrelated code. putc/getc are macros, not real calls, so wrapping
-// those symbols wouldn't reliably intercept them.
+// Only the process' original stdin/stdout/stderr are routed to the app's fds; every other stream
+// (real file I/O, including fwrite() to a file) goes straight to libc. fread() is not wrapped.
+// putc/getc are macros, not real calls, so wrapping those symbols wouldn't reliably intercept them.
 
 #if !defined(ESP_PLATFORM)
 
@@ -349,6 +349,7 @@ extern "C" {
 int __real_vfprintf(FILE* stream, const char* format, va_list args);
 int __real_fputs(const char* s, FILE* stream);
 int __real_fputc(int c, FILE* stream);
+size_t __real_fwrite(const void* data, size_t size, size_t count, FILE* stream);
 int __real_fgetc(FILE* stream);
 char* __real_fgets(char* buffer, int size, FILE* stream);
 }
@@ -372,6 +373,11 @@ int __real_fputc(int c, FILE* stream) {
     return real(c, stream);
 }
 
+size_t __real_fwrite(const void* data, size_t size, size_t count, FILE* stream) {
+    static auto real = reinterpret_cast<size_t (*)(const void*, size_t, size_t, FILE*)>(dlsym(RTLD_NEXT, "fwrite"));
+    return real(data, size, count, stream);
+}
+
 int __real_fgetc(FILE* stream) {
     static auto real = reinterpret_cast<int (*)(FILE*)>(dlsym(RTLD_NEXT, "fgetc"));
     return real(stream);
@@ -386,7 +392,8 @@ char* __real_fgets(char* buffer, int size, FILE* stream) {
 
 namespace {
 
-void writeAllTo(int fd, const void* data, size_t size) {
+/** @return the number of bytes written, less than `size` if the fd stopped accepting data */
+size_t writeAllTo(int fd, const void* data, size_t size) {
     const auto* bytes = static_cast<const char*>(data);
     size_t remaining = size;
     while (remaining > 0) {
@@ -397,6 +404,7 @@ void writeAllTo(int fd, const void* data, size_t size) {
         bytes += written;
         remaining -= static_cast<size_t>(written);
     }
+    return size - remaining;
 }
 
 int formatTo(int fd, const char* format, va_list args) {
@@ -425,24 +433,34 @@ int readOneFromStdin(char& out) {
     return static_cast<int>(app_io_read(STDIN_FILENO, &out, 1));
 }
 
+// The process' own streams, captured before anything can reassign stdin/stdout/stderr. A caller
+// that points stdout at a file (e.g. the shell's redirection) must get real file I/O, so only these
+// original streams are routed to the app's fds.
+FILE* const originalStdin = stdin;
+FILE* const originalStdout = stdout;
+FILE* const originalStderr = stderr;
+
 int targetFdOf(FILE* stream) {
-    if (stream == stdout) return STDOUT_FILENO;
-    if (stream == stderr) return STDERR_FILENO;
+    if (stream == originalStdout) return STDOUT_FILENO;
+    if (stream == originalStderr) return STDERR_FILENO;
     return -1;
 }
 
 } // namespace
 
 extern "C" {
 
+int __wrap_vfprintf(FILE* stream, const char* format, va_list args);
+int __wrap_fputc(int c, FILE* stream);
+
 int __wrap_vprintf(const char* format, va_list args) {
-    return formatTo(STDOUT_FILENO, format, args);
+    return __wrap_vfprintf(stdout, format, args);
 }
 
 int __wrap_printf(const char* format, ...) {
     va_list args;
     va_start(args, format);
-    int result = formatTo(STDOUT_FILENO, format, args);
+    int result = __wrap_vfprintf(stdout, format, args);
     va_end(args);
     return result;
 }
@@ -465,6 +483,9 @@ int __wrap_fprintf(FILE* stream, const char* format, ...) {
 }
 
 int __wrap_puts(const char* s) {
+    if (targetFdOf(stdout) < 0) {
+        return __real_fputs(s, stdout) < 0 ? EOF : __real_fputc('\n', stdout);
+    }
     writeAllTo(STDOUT_FILENO, s, strlen(s));
     writeAllTo(STDOUT_FILENO, "\n", 1);
     return 0;
@@ -480,9 +501,7 @@ int __wrap_fputs(const char* s, FILE* stream) {
 }
 
 int __wrap_putchar(int c) {
-    auto ch = static_cast<char>(c);
-    writeAllTo(STDOUT_FILENO, &ch, 1);
-    return c;
+    return __wrap_fputc(c, stdout);
 }
 
 int __wrap_fputc(int c, FILE* stream) {
@@ -495,20 +514,31 @@ int __wrap_fputc(int c, FILE* stream) {
     return __real_fputc(c, stream);
 }
 
-int __wrap_getchar() {
-    char c;
-    return readOneFromStdin(c) == 1 ? static_cast<unsigned char>(c) : EOF;
+size_t __wrap_fwrite(const void* data, size_t size, size_t count, FILE* stream) {
+    int fd = targetFdOf(stream);
+    if (fd < 0) {
+        return __real_fwrite(data, size, count, stream);
+    }
+    if (size == 0 || count == 0) {
+        return 0;
+    }
+    return writeAllTo(fd, data, size * count) / size;
 }
 
 int __wrap_fgetc(FILE* stream) {
-    if (stream == stdin) {
-        return __wrap_getchar();
+    if (stream != originalStdin) {
+        return __real_fgetc(stream);
     }
-    return __real_fgetc(stream);
+    char c;
+    return readOneFromStdin(c) == 1 ? static_cast<unsigned char>(c) : EOF;
+}
+
+int __wrap_getchar() {
+    return __wrap_fgetc(stdin);
 }
 
 char* __wrap_fgets(char* buffer, int size, FILE* stream) {
-    if (stream != stdin) {
+    if (stream != originalStdin) {
         return __real_fgets(buffer, size, stream);
     }
     if (size <= 0) {
@@ -544,6 +574,7 @@ TT_DYLD_INTERPOSE(__wrap_puts, puts)
 TT_DYLD_INTERPOSE(__wrap_fputs, fputs)
 TT_DYLD_INTERPOSE(__wrap_putchar, putchar)
 TT_DYLD_INTERPOSE(__wrap_fputc, fputc)
+TT_DYLD_INTERPOSE(__wrap_fwrite, fwrite)
 TT_DYLD_INTERPOSE(__wrap_getchar, getchar)
 TT_DYLD_INTERPOSE(__wrap_fgetc, fgetc)
 TT_DYLD_INTERPOSE(__wrap_fgets, fgets)
@@ -591,6 +622,10 @@ int fputc(int c, FILE* stream) {
     return __wrap_fputc(c, stream);
 }
 
+size_t fwrite(const void* data, size_t size, size_t count, FILE* stream) {
+    return __wrap_fwrite(data, size, count, stream);
+}
+
 int getchar() {
     return __wrap_getchar();
 }

Tactility/Private/Tactility/app/shell/Shell.h

@@ -58,8 +58,9 @@ void forEachCommand(void* context, void (*callback)(const Command& command, void
 /**
  * Completes the final word of `line`.
  *
- * The first word completes against command names, any later word against filesystem paths — which
- * is why this lives here rather than in LineEditor: it needs both the command table and ShellFs.
+ * Every word completes against filesystem paths, and a first word without a '/' against command
+ * names as well. That is why this lives here rather than in LineEditor: it needs both the command
+ * table and ShellFs.
  *
  * On a unique match, `outSuffix` receives the text to append. On several, the common prefix shared
  * by all of them is returned instead (which may be empty), and the candidates are printed.

Tactility/Private/Tactility/app/shell/shell/sh.h

@@ -6,6 +6,10 @@
 
 #include <stdio.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 // ---- shell state -----------------------------------------------------------
 
 struct node;   // forward (AST body of a function)
@@ -118,13 +122,6 @@ void sh_fields_init(sh_fields *f);
 void sh_fields_push(sh_fields *f, const char *s);
 void sh_fields_free(sh_fields *f);
 
-// Expand one raw word token into zero or more fields (word splitting applies to
-// unquoted expansions). Appends to `out`.
-void sh_expand_word(sh_state *st, const char *raw, sh_fields *out);
-// Expand a raw word to a single string (no splitting) - for redirect targets
-// and assignment values. Caller frees the returned string.
-char *sh_expand_single(sh_state *st, const char *raw);
-// Expand a here-doc body: parameter/command expansion and backslash escaping of
-// $ ` \ (double-quote semantics), preserving newlines and literal quotes.
-// Caller frees the returned string.
-char *sh_expand_heredoc(sh_state *st, const char *raw);
+#ifdef __cplusplus
+}
+#endif

Tactility/Private/Tactility/app/shell/shell/sh_arith.h

@@ -3,9 +3,17 @@
 
 #include "sh.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 // Evaluate arithmetic expression `expr` (signed long, C/POSIX operator set).
 // Returns 0 on success and writes *out; nonzero on error (division by zero,
 // bad token, etc.). `st` is used to read/write shell variables. On error, an
 // English message is left in *errmsg (points at a static buffer; may be NULL
 // on success). Pass errmsg=NULL to ignore.
 int sh_arith_eval(sh_state *st, const char *expr, long *out, const char **errmsg);
+
+#ifdef __cplusplus
+}
+#endif

Tactility/Private/Tactility/app/shell/shell/sh_builtins.h

@@ -1,6 +1,15 @@
 #pragma once
 #include "sh.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 // If argv[0] names a shell builtin, run it, store its status in *status, and
 // return 1. Otherwise return 0 (caller should try an external command).
+// `eval` and `source`/`.` run nested programs, so the executor handles those itself.
 int sh_run_builtin(sh_state *st, int argc, char **argv, int *status);
+
+#ifdef __cplusplus
+}
+#endif

Tactility/Private/Tactility/app/shell/shell/sh_glob.h

@@ -4,6 +4,10 @@
 // ${v#..}/${v%..} strip operators and, later, by `case` and pathname globbing.
 #pragma once
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 // Match string `s` against shell pattern `p` using '*' (any run, incl. empty)
 // and '?' (exactly one char). Backslash escapes the next metacharacter.
 // Returns 1 on match, 0 otherwise.
@@ -22,3 +26,7 @@ char *sh_strip_suffix(const char *s, const char *p, int longest);
 // the word literal, dash-style.
 #include "sh.h"
 int sh_glob_pathnames(const char *pat, sh_fields *out);
+
+#ifdef __cplusplus
+}
+#endif

Tactility/Private/Tactility/app/shell/shell/sh_lex.h

@@ -1,5 +1,9 @@
 #pragma once
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef enum {
     T_WORD,      // a raw word, quotes preserved (resolved later by sh_expand)
     T_SEMI,      // ;
@@ -38,3 +42,7 @@ typedef struct sh_toklist {
 // Tokenize src into tl. Returns 0 on success, -1 on lex error.
 int  sh_lex(const char *src, sh_toklist *tl);
 void sh_toklist_free(sh_toklist *tl);
+
+#ifdef __cplusplus
+}
+#endif