fs: Remove the special hack for pty in nx_vopen

let's replace the content of file in place instead

Signed-off-by: Xiang Xiao <xiaoxiang@xiaomi.com>
Change-Id: I538910d55815c7aec656c05dba4eab2fa1d6d964

drivers/serial/ptmx.c

@@ -166,9 +166,9 @@ static int ptmx_minor_allocate(void)
 
 static int ptmx_open(FAR struct file *filep)
 {
+  struct file temp;
   char devname[16];
   int minor;
-  int fd;
   int ret;
 
   /* Get exclusive access */
@@ -205,23 +205,25 @@ static int ptmx_open(FAR struct file *filep)
   /* Open the master device:  /dev/ptyN, where N=minor */
 
   snprintf(devname, 16, "/dev/pty%d", minor);
-  fd = nx_open(devname, O_RDWR);
-  DEBUGASSERT(fd >= 0);  /* nx_open() should never fail */
+  memcpy(&temp, filep, sizeof(temp));
+  ret = file_open(filep, devname, O_RDWR);
+  DEBUGASSERT(ret >= 0);  /* open() should never fail */
 
-  /* No unlink the master.  This will remove it from the VFS namespace,
+  /* Close the multiplexor device: /dev/ptmx */
+
+  ret = file_close(&temp);
+  DEBUGASSERT(ret >= 0);  /* close() should never fail */
+
+  /* Now unlink the master.  This will remove it from the VFS namespace,
    * the driver will still persist because of the open count on the
    * driver.
    */
 
-  ret = unlink(devname);
+  ret = nx_unlink(devname);
   DEBUGASSERT(ret >= 0);  /* unlink() should never fail */
-  UNUSED(ret);
-
-  /* Return the encoded, master file descriptor */
 
   nxsem_post(&g_ptmx.px_exclsem);
-  DEBUGASSERT((unsigned)fd <= OPEN_MAXFD);
-  return (int)OPEN_SETFD(fd);
+  return OK;
 
 errout_with_minor:
   ptmx_minor_free(minor);

fs/vfs/fs_open.c

@@ -244,41 +244,6 @@ int nx_vopen(FAR const char *path, int oflags, va_list ap)
       goto errout_with_fd;
     }
 
-#ifdef CONFIG_PSEUDOTERM_SUSV1
-  /* If the return value from the open method is > 0, then it may actually
-   * be an encoded file descriptor.  This kind of logic is currently only
-   * needed for /dev/ptmx:  When dev ptmx is opened, it does not return a
-   * file descriptor associated with the /dev/ptmx inode, but rather with
-   * the inode of master device created by the /dev/ptmx open method.
-   *
-   * The encoding supports (a) returning file descriptor 0 (which really
-   * should not happen), and (b) avoiding confusion if some other open
-   * method returns a positive, non-zero value which is not a file
-   * descriptor.
-   */
-
-  if (OPEN_ISFD(ret))
-    {
-      /* Release file descriptor and inode that we allocated.  We don't
-       * need those.
-       */
-
-      files_release(fd);
-      inode_release(inode);
-
-      /* Instead, decode and return the descriptor associated with the
-       * master side device.
-       */
-
-      fd = (int)OPEN_GETFD(ret);
-      DEBUGASSERT((unsigned)fd < (CONFIG_NFILE_DESCRIPTORS
-#ifdef CONFIG_NET
-                                  + CONFIG_NSOCKET_DESCRIPTORS
-#endif
-      ));
-    }
-#endif
-
   RELEASE_SEARCH(&desc);
   return fd;
 

include/nuttx/fs/fs.h

@@ -153,36 +153,6 @@
 #define DIRENT_SETPSEUDONODE(f) do (f) |= DIRENTFLAGS_PSEUDONODE; while (0)
 #define DIRENT_ISPSEUDONODE(f) (((f) & DIRENTFLAGS_PSEUDONODE) != 0)
 
-/* The struct file_operations open(0) normally returns zero on success and
- * a negated errno value on failure.  There is one case, however, where
- * the open method will redirect to another driver and return a file
- * descriptor instead.
- *
- * This case is when SUSv1 pseudo-terminals are used
- * (CONFIG_PSEUDOTERM_SUSV1=y).  In this case, the output is encoded and
- * decoded using these macros in order to support (a) returning file
- * descriptor 0 (which really should not happen), and (b) avoiding
- * confusion if some other open method returns a positive, non-zero value
- * hich is not a file descriptor.
- *
- *   OPEN_ISFD(r) tests if the return value from the open method is
- *     really a file descriptor.
- *   OPEN_SETFD(f) is used by an implementation of the open() method
- *     in order to encode a file descriptor in the return value.
- *   OPEN_GETFD(r) is use by the upper level open() logic to decode
- *     the file descriptor encoded in the return value.
- *
- * REVISIT: This only works for file descriptors in the in range 0-255.
- */
-
-#define OPEN_MAGIC      0x4200
-#define OPEN_MASK       0x00ff
-#define OPEN_MAXFD      0x00ff
-
-#define OPEN_ISFD(r)    (((r) & ~OPEN_MASK) == OPEN_MAGIC)
-#define OPEN_SETFD(f)   ((f) | OPEN_MAGIC)
-#define OPEN_GETFD(r)   ((r) & OPEN_MASK)
-
 /* nx_umount() is equivalent to nx_umount2() with flags = 0 */
 
 #define umount(t)       umount2(t,0)