Update

src/Form/Field/File.php

@@ -17,7 +17,10 @@ class File extends Field implements UploadFieldInterface
     /**
      * @var array
      */
-    protected $options = ['events' => []];
+    protected $options = [
+        'events' => [],
+        'override' => false,
+    ];
 
     public function __construct($column, $arguments = [])
     {

src/Form/Field/UploadField.php

@@ -98,8 +98,10 @@ trait UploadField
      */
     public function renameIfExists(UploadedFile $file)
     {
-        if ($this->getStorage()->exists("{$this->getDirectory()}/$this->name") && ! ($this->options['override'] ?? false)) {
+        if ($this->getStorage()->exists("{$this->getDirectory()}/$this->name") && ! $this->options['override']) {
             $this->name = $this->generateUniqueName($file);
+        } else {
+            $this->getStorage()->delete("{$this->getDirectory()}/$this->name");
         }
     }