Avoid ignoring files visible in Hive

Ignore in Hive only the files which have their names or the
names of their ancestor beginning with `.` or `_` characters.

Author: dprophet

plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/HiveFileIterator.java

@@ -150,7 +150,7 @@ static boolean isHiddenOrWithinHiddenParentDirectory(Path path, String prefix)
     {
         String pathString = path.toUri().getPath();
         checkArgument(pathString.startsWith(prefix), "path %s does not start with prefix %s", pathString, prefix);
-        return containsHiddenPathPartAfterIndex(pathString, prefix.length() + 1);
+        return containsHiddenPathPartAfterIndex(pathString, prefix.endsWith("/") ? prefix.length() : prefix.length() + 1);
     }
 
     @VisibleForTesting

plugin/trino-hive/src/test/java/io/trino/plugin/hive/fs/TestHiveFileIterator.java

@@ -30,6 +30,9 @@ public void testRelativeHiddenPathDetection()
         String root = new Path("file:///root-path").toUri().getPath();
         assertTrue(isHiddenOrWithinHiddenParentDirectory(new Path(root, ".hidden/child"), root));
         assertTrue(isHiddenOrWithinHiddenParentDirectory(new Path(root, "_hidden.txt"), root));
+        String rootWithSlash = new Path("file:///root-path/").toUri().getPath();
+        assertTrue(isHiddenOrWithinHiddenParentDirectory(new Path(rootWithSlash, ".hidden/child"), rootWithSlash));
+        assertTrue(isHiddenOrWithinHiddenParentDirectory(new Path(rootWithSlash, "_hidden.txt"), rootWithSlash));
         String rootWithinHidden = new Path("file:///root/.hidden/listing-root").toUri().getPath();
         assertFalse(isHiddenOrWithinHiddenParentDirectory(new Path(rootWithinHidden, "file.txt"), rootWithinHidden));
         String rootHiddenEnding = new Path("file:///root/hidden-ending_").toUri().getPath();

testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveHiddenFiles.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.hive;
+
+import com.google.inject.Inject;
+import io.trino.tempto.ProductTest;
+import io.trino.tempto.assertions.QueryAssert;
+import io.trino.tempto.hadoop.hdfs.HdfsClient;
+import org.assertj.core.api.Assertions;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import static io.trino.tempto.assertions.QueryAssert.Row.row;
+import static io.trino.tempto.assertions.QueryAssert.assertThat;
+import static io.trino.testing.TestingNames.randomNameSuffix;
+import static io.trino.tests.product.hive.util.TableLocationUtils.getTablePath;
+import static io.trino.tests.product.utils.QueryExecutors.onHive;
+import static io.trino.tests.product.utils.QueryExecutors.onTrino;
+
+public class TestHiveHiddenFiles
+        extends ProductTest
+{
+    @Inject
+    private HdfsClient hdfsClient;
+
+    @Test
+    public void testSelectFromTableContainingHiddenFiles()
+            throws Exception
+    {
+        String tableName = "test_table_hidden_files" + randomNameSuffix();
+        onTrino().executeQuery("CREATE TABLE " + tableName + " (col integer)");
+
+        onTrino().executeQuery("INSERT INTO " + tableName + " VALUES 1");
+        onTrino().executeQuery("INSERT INTO " + tableName + " VALUES 2");
+
+        List<QueryAssert.Row> tableRows = List.of(row(1), row(2));
+        assertThat(onTrino().executeQuery("SELECT * FROM " + tableName)).containsOnly(tableRows);
+        assertThat(onHive().executeQuery("SELECT * FROM " + tableName)).containsOnly(tableRows);
+
+        String tableLocation = getTablePath(tableName);
+        // Rename the table files to Hive hidden tableFiles (prefixed by `.` or `_` characters)
+        List<String> tableFiles = hdfsClient.listDirectory(tableLocation);
+        Assertions.assertThat(tableFiles).hasSize(2);
+        renameFile(tableLocation, tableFiles.get(0), '.' + tableFiles.get(0));
+        renameFile(tableLocation, tableFiles.get(1), '_' + tableFiles.get(1));
+
+        assertThat(onTrino().executeQuery("SELECT * FROM " + tableName)).hasNoRows();
+        assertThat(onHive().executeQuery("SELECT * FROM " + tableName)).hasNoRows();
+
+        onTrino().executeQuery("DROP TABLE IF EXISTS " + tableName);
+    }
+
+    @Test
+    public void testSelectFromTableContainingFilenamesWithUnderscore()
+            throws Exception
+    {
+        String tableName = "test_table_visible_underscore_files" + randomNameSuffix();
+        onTrino().executeQuery("CREATE TABLE " + tableName + " AS SELECT 1 AS col");
+
+        List<QueryAssert.Row> tableRows = List.of(row(1));
+        assertThat(onTrino().executeQuery("SELECT * FROM " + tableName)).containsOnly(tableRows);
+        assertThat(onHive().executeQuery("SELECT * FROM " + tableName)).containsOnly(tableRows);
+
+        String tableLocation = getTablePath(tableName);
+        // Prefix the table files with `f_` which should still keep them visible to Hive
+        for (String filename : hdfsClient.listDirectory(tableLocation)) {
+            // As long as the file is not hidden (starting with `.` or `_`), it should not be ignored by Hive
+            renameFile(tableLocation, filename, "f_" + filename);
+        }
+
+        assertThat(onTrino().executeQuery("SELECT * FROM " + tableName)).containsOnly(tableRows);
+        assertThat(onHive().executeQuery("SELECT * FROM " + tableName)).containsOnly(tableRows);
+
+        onTrino().executeQuery("DROP TABLE " + tableName);
+    }
+
+    private void renameFile(String directoryLocation, String filename, String newFilename)
+            throws IOException
+    {
+        try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
+            hdfsClient.loadFile(directoryLocation + "/" + filename, bos);
+            hdfsClient.saveFile(directoryLocation + "/" + newFilename, new ByteArrayInputStream(bos.toByteArray()));
+            hdfsClient.delete(directoryLocation + "/" + filename);
+        }
+    }
+}