/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.gravitino.spark.connector.integration.test.glue;

import com.google.common.base.Preconditions;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.gravitino.catalog.glue.GlueConstants;
import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfo;
import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfo.SparkColumnInfo;
import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfoChecker;
import org.apache.spark.sql.types.DataTypes;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Integration test for GravitinoGlueCatalog in Spark connector.
 *
 * <p>Tests mixed table type support (Hive format + Iceberg format) in a single Glue database. Uses
 * Moto server to mock AWS Glue API, similar to MotoGlueCatalogIT in the server module.
 */
public abstract class SparkGlueCatalogIT extends SparkGlueEnvIT {

  private static final Logger LOG = LoggerFactory.getLogger(SparkGlueCatalogIT.class);

  private String glueEndpoint;
  private String awsRegion;
  private String awsAccessKeyId;
  private String awsSecretAccessKey;

  @Override
  protected String getCatalogName() {
    return "glue";
  }

  @Override
  protected String getProvider() {
    return "glue";
  }

  @Override
  protected Map<String, String> getCatalogConfigs() {
    Preconditions.checkArgument(
        awsRegion != null, "awsRegion must be set before getCatalogConfigs()");
    Preconditions.checkArgument(
        awsAccessKeyId != null, "awsAccessKeyId must be set before getCatalogConfigs()");
    Preconditions.checkArgument(
        awsSecretAccessKey != null, "awsSecretAccessKey must be set before getCatalogConfigs()");
    Map<String, String> catalogProperties = new java.util.HashMap<>();
    catalogProperties.put(GlueConstants.AWS_REGION, awsRegion);
    catalogProperties.put(GlueConstants.AWS_ACCESS_KEY_ID, awsAccessKeyId);
    catalogProperties.put(GlueConstants.AWS_SECRET_ACCESS_KEY, awsSecretAccessKey);
    catalogProperties.put(GlueConstants.WAREHOUSE, warehouse);
    if (glueEndpoint != null) {
      catalogProperties.put(GlueConstants.AWS_GLUE_ENDPOINT, glueEndpoint);
    }
    return catalogProperties;
  }

  @Override
  protected boolean supportsSparkSQLClusteredBy() {
    return true;
  }

  @Override
  protected boolean supportsPartition() {
    return true;
  }

  @Override
  protected boolean supportsDelete() {
    return false;
  }

  @Override
  protected boolean supportsSchemaEvolution() {
    return false;
  }

  @Override
  protected boolean supportsReplaceColumns() {
    return false;
  }

  @Override
  protected boolean supportsSchemaAndTableProperties() {
    return true;
  }

  @Override
  protected boolean supportsComplexType() {
    // Glue does not support Gravitino complex types (LIST, MAP, STRUCT) in table columns.
    return false;
  }

  @Override
  protected boolean supportsUpdateColumnPosition() {
    return false;
  }

  @Override
  protected boolean supportsFunction() {
    return false;
  }

  @Override
  protected boolean supportsIcebergPartitionTransforms() {
    return true;
  }

  @BeforeAll
  @Override
  protected void startUp() throws Exception {
    String accessKeyId = System.getenv("AWS_ACCESS_KEY_ID");
    String secretAccessKey = System.getenv("AWS_SECRET_ACCESS_KEY");
    setGlueEndpoint(null);
    setAwsCredentials(accessKeyId, secretAccessKey);
    setAwsRegion(System.getenv("AWS_DEFAULT_REGION"));
    setS3Credentials(null, accessKeyId, secretAccessKey);
    setS3BucketName(System.getenv("AWS_S3_TEST_BUCKET"));
    super.startUp();
  }

  /**
   * Sets the Glue endpoint for testing. Called by subclasses after Moto container is started.
   *
   * @param endpoint the Glue API endpoint URL
   */
  protected void setGlueEndpoint(String endpoint) {
    this.glueEndpoint = endpoint;
  }

  protected void setAwsRegion(String region) {
    this.awsRegion = region;
  }

  /**
   * Sets AWS credentials for testing.
   *
   * @param accessKeyId AWS access key ID
   * @param secretAccessKey AWS secret access key
   */
  protected void setAwsCredentials(String accessKeyId, String secretAccessKey) {
    this.awsAccessKeyId = accessKeyId;
    this.awsSecretAccessKey = secretAccessKey;
  }

  @Override
  protected String getDefaultAwsRegion() {
    return awsRegion;
  }

  @Override
  protected String getGlueEndpoint() {
    return glueEndpoint;
  }

  protected String getAwsRegion() {
    return awsRegion;
  }

  /**
   * Overrides to use CASCADE so that databases with stale tables (e.g., from prior test runs) can
   * be cleaned up. Glue tables persist across JVM restarts and may be left behind after a crash.
   */
  @Override
  protected void dropDatabaseIfExists(String database) {
    sql("DROP DATABASE IF EXISTS " + database + " CASCADE");
  }

  /**
   * Overrides to always use PARQUET format for Glue. Without an explicit USING clause, Spark may
   * route CREATE TABLE through the V1 Hive path, bypassing the location-derivation logic in
   * GravitinoGlueCatalog.createTable and leaving tables without a stored S3 location.
   */
  @Override
  protected void createSimpleTable(String identifier) {
    sql(getCreateSimpleTableString(identifier) + " USING PARQUET");
  }

  /**
   * Overrides to recreate the database with the correct S3 location. The base implementation uses
   * CREATE DATABASE IF NOT EXISTS with a local HDFS path (/user/hive/db), causing tables to inherit
   * a local path as their default location. We drop and recreate to ensure the S3 location is
   * always set correctly, so stale data cleanup via dropTableIfExists works reliably.
   */
  @Override
  protected void createDatabaseIfNotExists(String database, String provider) {
    String dbLocation = warehouse + "/" + database;
    dropDatabaseIfExists(database);
    // Delete S3 data directory so stale data files from prior runs don't cause duplicate rows.
    deleteDirIfExists(dbLocation);
    sql(String.format("CREATE DATABASE %s LOCATION '%s'", database, dbLocation));
  }

  /**
   * Overrides to also delete the S3 data directory after dropping the table. Unlike HDFS managed
   * tables, Glue external tables do not delete S3 data on DROP TABLE. Stale data files cause
   * duplicate rows on the next create+insert cycle.
   */
  @Override
  protected void dropTableIfExists(String tableName) {
    String location = null;
    try {
      location = getTableInfo(tableName).getTableLocation();
    } catch (Exception e) {
      // Table may not exist yet — location stays null, nothing to delete.
      LOG.debug("Could not get location for table {}: {}", tableName, e.getMessage());
    }
    super.dropTableIfExists(tableName);
    if (location != null) {
      deleteDirIfExists(location);
    }
  }

  /**
   * Overrides base class: use USING PARQUET to ensure the table goes through the Gravitino Glue
   * catalog (V2 path).
   */
  @Test
  @Override
  protected void testDropAndWriteTable() {
    String tableName = "drop_then_create_write_table";
    dropTableIfExists(tableName);
    sql(getCreateSimpleTableString(tableName) + " USING PARQUET");
    SparkTableInfo info = getTableInfo(tableName);
    checkTableReadWrite(info);

    // External tables on S3 do not delete data on DROP TABLE; clean up explicitly.
    String location = info.getTableLocation();
    dropTableIfExists(tableName);
    if (location != null) {
      deleteDirIfExists(location);
    }

    sql(getCreateSimpleTableString(tableName) + " USING PARQUET");
    checkTableReadWrite(getTableInfo(tableName));
  }

  @Disabled("Glue does not support table rename for non-Iceberg tables")
  @Test
  @Override
  protected void testRenameTable() {}

  // -------------------------------------------------------------------------
  // Test mixed table types (Hive format + Iceberg format)
  // -------------------------------------------------------------------------

  @Test
  void testCreateHiveFormatTable() {
    String tableName = "test_hive_format_table";
    dropTableIfExists(tableName);
    String createTableSql = getCreateSimpleTableString(tableName);
    createTableSql += " USING PARQUET";
    sql(createTableSql);

    SparkTableInfo tableInfo = getTableInfo(tableName);
    SparkTableInfoChecker checker =
        SparkTableInfoChecker.create().withName(tableName).withColumns(getSimpleTableColumn());
    checker.check(tableInfo);
    checkTableReadWrite(tableInfo);
  }

  @Test
  void testCreateIcebergFormatTable() {
    String tableName = "test_iceberg_format_table";
    dropTableIfExists(tableName);
    String createTableSql = getCreateSimpleTableString(tableName);
    createTableSql += " USING iceberg";
    sql(createTableSql);

    SparkTableInfo tableInfo = getTableInfo(tableName);
    SparkTableInfoChecker checker =
        SparkTableInfoChecker.create().withName(tableName).withColumns(getSimpleTableColumn());
    checker.check(tableInfo);
    checkTableReadWrite(tableInfo);
  }

  @Test
  void testMixedTableTypesInSameDatabase() {
    String hiveTable = "mixed_hive_table";
    String icebergTable = "mixed_iceberg_table";

    // Create non-partitioned Hive format table
    dropTableIfExists(hiveTable);
    sql(getCreateSimpleTableString(hiveTable) + " USING PARQUET");

    // Create non-partitioned Iceberg format table
    dropTableIfExists(icebergTable);
    sql(getCreateSimpleTableString(icebergTable) + " USING iceberg");

    // Both tables should be accessible
    SparkTableInfo hiveTableInfo = getTableInfo(hiveTable);
    SparkTableInfoChecker hiveChecker =
        SparkTableInfoChecker.create().withName(hiveTable).withColumns(getSimpleTableColumn());
    hiveChecker.check(hiveTableInfo);
    checkTableReadWrite(hiveTableInfo);

    SparkTableInfo icebergTableInfo = getTableInfo(icebergTable);
    SparkTableInfoChecker icebergChecker =
        SparkTableInfoChecker.create().withName(icebergTable).withColumns(getSimpleTableColumn());
    icebergChecker.check(icebergTableInfo);
    checkTableReadWrite(icebergTableInfo);
  }

  @Test
  void testHivePartitionedTable() {
    String tableName = "test_hive_partitioned_table";
    dropTableIfExists(tableName);
    // Use existing columns as partition keys (datasource-style) so partition columns stay in
    // schema.
    // Spark places partition columns last; columns = [id, age, name] with name as partition.
    sql(
        "CREATE TABLE "
            + tableName
            + " (id INT COMMENT 'id comment', age INT, name STRING COMMENT '') USING PARQUET"
            + " PARTITIONED BY (name)");

    SparkTableInfo tableInfo = getTableInfo(tableName);
    SparkTableInfoChecker checker =
        SparkTableInfoChecker.create()
            .withName(tableName)
            .withColumns(
                Arrays.asList(
                    SparkColumnInfo.of("id", DataTypes.IntegerType, "id comment"),
                    SparkColumnInfo.of("age", DataTypes.IntegerType, null),
                    SparkColumnInfo.of("name", DataTypes.StringType, "")))
            .withIdentifyPartition(Arrays.asList("name"));
    checker.check(tableInfo);
    checkTableReadWrite(tableInfo);
  }

  @Test
  void testIcebergPartitionedTable() {
    String tableName = "test_iceberg_partitioned_table";
    dropTableIfExists(tableName);
    // Partition by an existing column (id). Iceberg stores partition columns separately from
    // schema.
    sql(getCreateSimpleTableString(tableName) + " USING iceberg PARTITIONED BY (id)");

    SparkTableInfo tableInfo = getTableInfo(tableName);
    SparkTableInfoChecker checker =
        SparkTableInfoChecker.create().withName(tableName).withColumns(getSimpleTableColumn());
    checker.check(tableInfo);
    checkTableReadWrite(tableInfo);
  }

  @Test
  void testInsertHiveTable() {
    String tableName = "test_insert_hive_table";
    dropTableIfExists(tableName);
    String createTableSql = getCreateSimpleTableString(tableName);
    createTableSql += " USING PARQUET";
    sql(createTableSql);

    sql(String.format("INSERT INTO %s VALUES (1, 'name1', 25)", tableName));
    List<String> tableData = getTableData(tableName);
    Assertions.assertFalse(tableData.isEmpty());
    Assertions.assertEquals("1,name1,25", tableData.get(0));
  }

  @Test
  void testInsertIcebergTable() {
    String tableName = "test_insert_iceberg_table";
    dropTableIfExists(tableName);
    String createTableSql = getCreateSimpleTableString(tableName);
    createTableSql += " USING iceberg";
    sql(createTableSql);

    sql(String.format("INSERT INTO %s VALUES (1, 'name1', 25)", tableName));
    List<String> tableData = getTableData(tableName);
    Assertions.assertFalse(tableData.isEmpty());
    Assertions.assertEquals("1,name1,25", tableData.get(0));
  }

  /**
   * Regression test for https://github.com/apache/gravitino/issues/11534.
   *
   * <p>After ALTER TABLE ADD COLUMNS on an Iceberg table, querying the new column must succeed.
   * Previously, icebergGlueCatalog's CachingCatalog was never invalidated after alterTable, causing
   * {@code IllegalStateException: Couldn't find <newCol> in [<oldCols>]}.
   */
  @Test
  void testIcebergAlterTableAddColumnCacheInvalidation() {
    String tableName = "test_iceberg_alter_cache";
    dropTableIfExists(tableName);
    sql(
        String.format(
            "CREATE TABLE %s (id INT COMMENT 'id', name STRING COMMENT 'name') USING iceberg",
            tableName));
    sql(String.format("INSERT INTO %s VALUES (1, 'Alice')", tableName));

    // Warm up the Iceberg SparkCatalog's CachingCatalog with an initial read
    List<String> before = getTableData(tableName);
    Assertions.assertEquals(1, before.size());
    Assertions.assertEquals("1,Alice", before.get(0));

    // Add a column — must invalidate icebergGlueCatalog cache
    sql(String.format("ALTER TABLE %s ADD COLUMNS (age INT)", tableName));

    // Before the fix this threw:
    //   IllegalStateException: Couldn't find age#N in [id#N, name#N]
    List<String> after = getQueryData(String.format("SELECT id, name, age FROM %s", tableName));
    Assertions.assertEquals(1, after.size());
    Assertions.assertEquals("1,Alice,NULL", after.get(0));
  }

  @Test
  void testRenameIcebergTable() {
    String oldName = "test_iceberg_rename_old";
    String newName = "test_iceberg_rename_new";
    dropTableIfExists(oldName);
    dropTableIfExists(newName);

    sql(getCreateSimpleTableString(oldName) + " USING iceberg");
    sql(String.format("INSERT INTO %s VALUES (1, 'before_rename', 10)", oldName));

    Assertions.assertTrue(tableExists(oldName));
    Assertions.assertFalse(tableExists(newName));

    sql(String.format("ALTER TABLE %s RENAME TO %s", oldName, newName));

    Assertions.assertFalse(tableExists(oldName));
    Assertions.assertTrue(tableExists(newName));

    List<String> tableData = getTableData(newName);
    Assertions.assertFalse(tableData.isEmpty());
    Assertions.assertEquals("1,before_rename,10", tableData.get(0));
  }

  @Test
  void testCreateTableWithComment() {
    String tableName = "test_table_with_comment";
    dropTableIfExists(tableName);
    String createTableSql = getCreateSimpleTableString(tableName);
    createTableSql += " USING PARQUET COMMENT 'Test table comment'";
    sql(createTableSql);

    SparkTableInfo tableInfo = getTableInfo(tableName);
    Assertions.assertEquals("Test table comment", tableInfo.getComment());
    checkTableReadWrite(tableInfo);
  }

  @Test
  void testExternalTableLocation() {
    String tableName = "test_external_table";
    dropTableIfExists(tableName);
    String externalLocation = warehouse + "/external_glue_db/external_table";
    deleteDirIfExists(externalLocation);

    String createTableSql = getCreateSimpleTableString(tableName);
    createTableSql += String.format(" USING PARQUET LOCATION '%s'", externalLocation);
    sql(createTableSql);

    SparkTableInfo tableInfo = getTableInfo(tableName);
    Assertions.assertEquals(externalLocation, tableInfo.getTableLocation());
    checkTableReadWrite(tableInfo);
  }

  @Test
  @Override
  protected void testLoadCatalogs() {
    // Glue catalog is not shown in SHOW CATALOGS output (Gravitino registers it lazily via Spark
    // plugin). Verify accessibility by listing databases instead.
    Assertions.assertDoesNotThrow(() -> sql("SHOW DATABASES IN " + getCatalogName()));
  }

  /**
   * Overrides base: skips S3 directory verification when no explicit LOCATION was given. Glue does
   * not store the auto-assigned warehouse location in table properties, so we cannot reconstruct
   * the exact S3 path. Data read/write correctness is already validated by {@link
   * #checkTableReadWrite}.
   */
  @Override
  protected void checkPartitionDirExists(SparkTableInfo table) {
    if (table.getTableLocation() == null) {
      return;
    }
    super.checkPartitionDirExists(table);
  }

  // -------------------------------------------------------------------------
  // Override exception-assertion tests (Glue throws different exception types)
  // -------------------------------------------------------------------------

  /**
   * Override: Glue does not support local filesystem paths for database locations; use S3 path.
   * Also, Glue does not return Owner as "anonymous".
   */
  @Test
  @Override
  protected void testCreateAndLoadSchema() {
    String testDatabaseName = "t_create1";
    dropDatabaseIfExists(testDatabaseName);
    sql("CREATE DATABASE " + testDatabaseName + " WITH DBPROPERTIES (ID=001);");
    Map<String, String> databaseMeta = getDatabaseMetadata(testDatabaseName);
    // Glue does not auto-assign a location when none is specified, so no "Location" row appears
    String properties = databaseMeta.get("Properties");
    Assertions.assertTrue(properties.contains("(ID,001)"));

    testDatabaseName = "t_create2";
    dropDatabaseIfExists(testDatabaseName);
    String testDatabaseLocation = warehouse + "/" + testDatabaseName;
    sql(
        String.format(
            "CREATE DATABASE %s COMMENT 'comment' LOCATION '%s' WITH DBPROPERTIES (ID=002);",
            testDatabaseName, testDatabaseLocation));
    databaseMeta = getDatabaseMetadata(testDatabaseName);
    String comment = databaseMeta.get("Comment");
    Assertions.assertEquals("comment", comment);
    Assertions.assertTrue(databaseMeta.get("Location").contains(testDatabaseName));
    properties = databaseMeta.get("Properties");
    Assertions.assertTrue(properties.contains("(ID,002)"));
  }

  /**
   * Override: Glue may throw AnalysisException instead of NoSuchNamespaceException when listing
   * tables in a nonexistent schema.
   */
  @Test
  void testListTables() {
    String tableName = "t_list";
    Set<String> tableNames = listTableNames();
    Assertions.assertFalse(tableNames.contains(tableName));
    createSimpleTable(tableName);
    tableNames = listTableNames();
    Assertions.assertTrue(tableNames.contains(tableName));
    // Glue throws AnalysisException or other runtime exception through Spark's Hive catalog adapter
    Assertions.assertThrows(Exception.class, () -> sql("SHOW TABLES IN nonexistent_schema"));
  }

  /**
   * Override: Glue may throw AnalysisException instead of NoSuchNamespaceException when altering a
   * nonexistent schema.
   */
  @Test
  @Override
  protected void testAlterSchema() {
    String testDatabaseName = "t_alter";
    dropDatabaseIfExists(testDatabaseName);
    sql("CREATE DATABASE " + testDatabaseName + " WITH DBPROPERTIES (ID=001);");
    Assertions.assertTrue(
        getDatabaseMetadata(testDatabaseName).get("Properties").contains("(ID,001)"));

    sql(String.format("ALTER DATABASE %s SET DBPROPERTIES ('ID'='002')", testDatabaseName));
    Assertions.assertFalse(
        getDatabaseMetadata(testDatabaseName).get("Properties").contains("(ID,001)"));
    Assertions.assertTrue(
        getDatabaseMetadata(testDatabaseName).get("Properties").contains("(ID,002)"));

    // Glue may throw AnalysisException instead of NoSuchNamespaceException
    Assertions.assertThrows(
        Exception.class, () -> sql("ALTER DATABASE notExists SET DBPROPERTIES ('ID'='001')"));
  }

  /**
   * Override: Glue may throw AnalysisException instead of NoSuchNamespaceException when dropping a
   * nonexistent schema.
   */
  @Test
  @Override
  protected void testDropSchema() {
    String testDatabaseName = "t_drop";
    dropDatabaseIfExists(testDatabaseName);
    Set<String> databases = getDatabases();
    Assertions.assertFalse(databases.contains(testDatabaseName));

    sql("CREATE DATABASE " + testDatabaseName);
    databases = getDatabases();
    Assertions.assertTrue(databases.contains(testDatabaseName));

    sql("DROP DATABASE " + testDatabaseName);
    databases = getDatabases();
    Assertions.assertFalse(databases.contains(testDatabaseName));

    // Glue may throw AnalysisException instead of NoSuchNamespaceException
    Assertions.assertThrows(Exception.class, () -> sql("DROP DATABASE notExists"));
  }

  /**
   * Override: Glue may throw AnalysisException instead of NoSuchNamespaceException when listing
   * tables from a nonexistent database.
   */
  @Test
  @Override
  protected void testListTable() {
    String table1 = "list1";
    String table2 = "list2";
    createSimpleTable(table1);
    createSimpleTable(table2);
    Set<String> tables = listTableNames();
    Assertions.assertTrue(tables.contains(table1));
    Assertions.assertTrue(tables.contains(table2));

    String database = "db_list";
    String table3 = "list3";
    String table4 = "list4";
    createDatabaseIfNotExists(database, getProvider());
    dropTableIfExists(String.join(".", database, table3));
    dropTableIfExists(String.join(".", database, table4));
    createSimpleTable(String.join(".", database, table3));
    createSimpleTable(String.join(".", database, table4));
    tables = listTableNames(database);

    Assertions.assertTrue(tables.contains(table3));
    Assertions.assertTrue(tables.contains(table4));

    // Glue may throw AnalysisException instead of NoSuchNamespaceException
    Assertions.assertThrows(Exception.class, () -> listTableNames("not_exists_db"));
  }
}
