/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.kafka.connect.file.integration;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.connect.file.FileStreamSourceConnector;
import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster;
import org.apache.kafka.test.TestUtils;

import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import static org.apache.kafka.connect.file.FileStreamSourceConnector.FILE_CONFIG;
import static org.apache.kafka.connect.file.FileStreamSourceConnector.TOPIC_CONFIG;
import static org.apache.kafka.connect.file.FileStreamSourceTask.FILENAME_FIELD;
import static org.apache.kafka.connect.file.FileStreamSourceTask.POSITION_FIELD;
import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
import static org.junit.jupiter.api.Assertions.assertEquals;

@Tag("integration")
public class FileStreamSourceConnectorIntegrationTest {

    private static final String CONNECTOR_NAME = "test-connector";
    private static final String TOPIC = "test-topic";
    private static final String LINE_FORMAT = "Line %d";
    private static final int NUM_LINES = 5;
    private static final long TIMEOUT_MS = TimeUnit.SECONDS.toMillis(15);
    private final EmbeddedConnectCluster connect = new EmbeddedConnectCluster.Builder().build();
    private File sourceFile;

    @BeforeEach
    public void setup() throws Exception {
        connect.start();
        sourceFile = createTempFile(NUM_LINES);
        connect.kafka().createTopic(TOPIC);
    }

    @AfterEach
    public void tearDown() {
        connect.stop();
    }

    @Test
    public void testSimpleSource() throws Exception {
        Map<String, String> connectorConfigs = baseConnectorConfigs(sourceFile.getAbsolutePath());
        connect.configureConnector(CONNECTOR_NAME, connectorConfigs);
        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, 1,
            "Connector and task did not start in time");

        int i = 0;
        for (ConsumerRecord<byte[], byte[]> record : connect.kafka().consume(NUM_LINES, TIMEOUT_MS, TOPIC)) {
            assertEquals(String.format(LINE_FORMAT, i++), new String(record.value()));
        }
    }

    @Test
    public void testStopResumeSavedOffset() throws Exception {
        Map<String, String> connectorConfigs = baseConnectorConfigs(sourceFile.getAbsolutePath());
        connect.configureConnector(CONNECTOR_NAME, connectorConfigs);
        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, 1,
            "Connector and task did not start in time");

        // Wait for the initially written records to be sourced by the connector and produced to the configured Kafka topic
        connect.kafka().consume(NUM_LINES, TIMEOUT_MS, TOPIC);

        connect.stopConnector(CONNECTOR_NAME);
        connect.assertions().assertConnectorIsStopped(CONNECTOR_NAME, "Connector did not stop in time");

        // Append NUM_LINES more lines to the file
        try (PrintStream printStream = new PrintStream(Files.newOutputStream(sourceFile.toPath(), StandardOpenOption.APPEND))) {
            for (int i = NUM_LINES; i < 2 * NUM_LINES; i++) {
                printStream.printf(LINE_FORMAT + "%n", i);
            }
        }

        connect.resumeConnector(CONNECTOR_NAME);
        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, 1,
            "Connector and task did not resume in time");

        int i = 0;
        for (ConsumerRecord<byte[], byte[]> record : connect.kafka().consume(2 * NUM_LINES, TIMEOUT_MS, TOPIC)) {
            assertEquals(String.format(LINE_FORMAT, i++), new String(record.value()));
        }

        // We expect exactly (2 * NUM_LINES) messages to be produced since the connector should continue from where it left off on being resumed.
        // We verify this by consuming all the messages from the topic after we've already ensured that at least (2 * NUM_LINES) messages can be
        // consumed above.
        assertEquals(2 * NUM_LINES, connect.kafka().consumeAll(TIMEOUT_MS, TOPIC).count());
    }

    @Test
    public void testAlterOffsets() throws Exception {
        Map<String, String> connectorConfigs = baseConnectorConfigs(sourceFile.getAbsolutePath());
        connect.configureConnector(CONNECTOR_NAME, connectorConfigs);
        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, 1,
            "Connector and task did not start in time");

        // Wait for the initially written records to be sourced by the connector and produced to the configured Kafka topic
        connect.kafka().consume(NUM_LINES, TIMEOUT_MS, TOPIC);

        connect.stopConnector(CONNECTOR_NAME);
        connect.assertions().assertConnectorIsStopped(CONNECTOR_NAME, "Connector did not stop in time");

        // Alter the offsets to make the connector re-process the last line in the file
        connect.alterSourceConnectorOffset(
            CONNECTOR_NAME,
            Collections.singletonMap(FILENAME_FIELD, sourceFile.getAbsolutePath()),
            Collections.singletonMap(POSITION_FIELD, 28L)
        );

        connect.resumeConnector(CONNECTOR_NAME);
        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, 1,
            "Connector and task did not resume in time");

        Iterator<ConsumerRecord<byte[], byte[]>> recordIterator = connect.kafka().consume(NUM_LINES + 1, TIMEOUT_MS, TOPIC).iterator();

        for (int i = 0; i < NUM_LINES; i++) {
            assertEquals(String.format(LINE_FORMAT, i), new String(recordIterator.next().value()));
        }

        // Verify that the last line has been sourced again after the alter offsets request
        assertEquals(String.format(LINE_FORMAT, NUM_LINES - 1), new String(recordIterator.next().value()));
    }

    @Test
    public void testResetOffsets() throws Exception {
        Map<String, String> connectorConfigs = baseConnectorConfigs(sourceFile.getAbsolutePath());
        connect.configureConnector(CONNECTOR_NAME, connectorConfigs);
        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, 1,
            "Connector and task did not start in time");

        // Wait for the initially written records to be sourced by the connector and produced to the configured Kafka topic
        connect.kafka().consume(NUM_LINES, TIMEOUT_MS, TOPIC);

        connect.stopConnector(CONNECTOR_NAME);
        connect.assertions().assertConnectorIsStopped(CONNECTOR_NAME, "Connector did not stop in time");

        // Reset the offsets to make the connector re-read all the previously written lines
        connect.resetConnectorOffsets(CONNECTOR_NAME);

        connect.resumeConnector(CONNECTOR_NAME);
        connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, 1,
            "Connector and task did not resume in time");

        Iterator<ConsumerRecord<byte[], byte[]>> recordIterator = connect.kafka().consume(2 * NUM_LINES, TIMEOUT_MS, TOPIC).iterator();

        int i = 0;
        while (i < NUM_LINES) {
            assertEquals(String.format(LINE_FORMAT, i++), new String(recordIterator.next().value()));
        }

        // Verify that the same lines have been sourced again after the reset offsets request
        while (i < 2 * NUM_LINES) {
            assertEquals(String.format(LINE_FORMAT, i - NUM_LINES), new String(recordIterator.next().value()));
            i++;
        }

        // We expect exactly (2 * NUM_LINES) messages to be produced since the connector should reprocess exactly the same NUM_LINES messages after
        // the offsets have been reset. We verify this by consuming all the messages from the topic after we've already ensured that at least
        // (2 * NUM_LINES) messages can be consumed above.
        assertEquals(2 * NUM_LINES, connect.kafka().consumeAll(TIMEOUT_MS, TOPIC).count());
    }

    /**
     * Create a temporary file and append {@code numLines} to it
     *
     * @param numLines the number of lines to be appended to the created file
     * @return the created file
     */
    private File createTempFile(int numLines) throws Exception {
        File sourceFile = TestUtils.tempFile();

        try (PrintStream printStream = new PrintStream(Files.newOutputStream(sourceFile.toPath()))) {
            for (int i = 0; i < numLines; i++) {
                printStream.printf(LINE_FORMAT + "%n", i);
            }
        }

        return sourceFile;
    }

    private Map<String, String> baseConnectorConfigs(String filePath) {
        Map<String, String> connectorConfigs = new HashMap<>();
        connectorConfigs.put(CONNECTOR_CLASS_CONFIG, FileStreamSourceConnector.class.getName());
        connectorConfigs.put(TOPIC_CONFIG, TOPIC);
        connectorConfigs.put(FILE_CONFIG, filePath);
        return connectorConfigs;
    }
}
