是否可以执行javacron工作来导出BigQuery表

Question

我想上传一个java cron作业，以便进行一些查询，并每周将一个表从BigQuery导出到Google Storage。为此，我使用了Eclipse的Google插件将cron上传到AppEngine。

问题是，我的java cron作业调用了一个具有google maven依赖关系的java类来访问BigQuery，但当cron上传到AppEngine时，会出现以下错误：

Error for /cron/gaejcronjob 
java.lang.NoClassDefFoundError: com/google/api/client/json/JsonFactory

我读过这个问题：java.lang.ClassNotFoundException：com.google.api.client.json.JsonFactory，但它的响应并不能解决问题。

编辑：（添加pom.xml、GAEJCronServlet.java和BigQuery.java代码）

pom.xml：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>XTG.Cron.Jobs</groupId>
    <artifactId>BigQuery</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <build>
        <sourceDirectory>src</sourceDirectory>
        <resources>
            <resource>
                <directory>src</directory>
                <excludes>
                    <exclude>**/*.java</exclude>
                </excludes>
            </resource>
        </resources>
        <plugins>
            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.3</version>
                <configuration>
                    <source>1.7</source>
                    <target>1.7</target>
                </configuration>
            </plugin>
        </plugins>
    </build>
    <dependencies>
        <dependency>
            <groupId>com.google.cloud.dataflow</groupId>
            <artifactId>google-cloud-dataflow-java-sdk-all</artifactId>
            <version>LATEST</version>
        </dependency>
    </dependencies>
</project>

GAEJCronServlet.java：

package com.gaejexperiments.cron;
import java.io.IOException;
import java.util.logging.Logger;
import javax.servlet.ServletException;
import javax.servlet.http.*;
import com.gaejexperiments.cron.BigQuery;
@SuppressWarnings("serial")
public class GAEJCronServlet extends HttpServlet {
    private static final Logger _logger = Logger.getLogger(GAEJCronServlet.class.getName());
    public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
        try {
            _logger.info("Cron Job has been executed");
            BigQuery bigquery = new BigQuery();
            bigquery.exportTable();
        } catch (Exception ex) {
            //_logger.info(ex);
        }
    }
    @Override
    public void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
        doGet(req, resp);
    }
}

BigQuery.java：

package com.gaejexperiments.cron;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.ErrorProto;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobConfiguration;
import com.google.api.services.bigquery.model.JobConfigurationExtract;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.TableReference;
public class BigQuery {
    private final String PROJECT_ID = projectId;
    private final String DATASET_ID = "bigquerytest";
    private final String TABLE_ID = "test";
    private Bigquery service = null;
    public void main(String[] args) {
        try {
            HttpTransport httpTransport = new NetHttpTransport();
            JsonFactory jsonFactory = new JacksonFactory();
            GoogleCredential credential = GoogleCredential.getApplicationDefault(httpTransport, jsonFactory);
            Bigquery.Builder serviceBuilder = 
                new Bigquery.Builder(httpTransport, jsonFactory, credential)
                .setApplicationName("Bigquery ");
            service = serviceBuilder.build();
            if (service == null || service.jobs() == null) {
                throw new Exception("Service is null");
            }
        }
        catch (Exception ex) {
            System.out.println("Caught exception: " + ex + "n");
            ex.printStackTrace();
            System.exit(1);
        }
        System.exit(0);
    }
    public void exportTable() throws Exception{
        //Export
        TableReference sourceTable = new TableReference();
        sourceTable.setProjectId(PROJECT_ID);
        sourceTable.setDatasetId(DATASET_ID);
        sourceTable.setTableId(TABLE_ID);
        JobConfigurationExtract jobExtract = new JobConfigurationExtract();
        jobExtract.setDestinationFormat("CSV");
        jobExtract.setDestinationUri("gs://xtg-bigquery/test1.csv");
        jobExtract.setSourceTable(sourceTable);
        JobConfiguration jobConfig = new JobConfiguration();
        jobConfig.setExtract(jobExtract);
        JobReference jobRef = new JobReference();
        jobRef.setProjectId(PROJECT_ID);
        Job outputJob = new Job();
        outputJob.setConfiguration(jobConfig);
        outputJob.setJobReference(jobRef);
        Job job = service.jobs().insert(PROJECT_ID, 
                                        outputJob).execute();
        if (job == null) {
            throw new Exception("Job is null");
        }
        while (true) {
            String status = job.getStatus().getState();
            if (status != null || ("DONE").equalsIgnoreCase(status)) {
                break;
            }
            Thread.sleep(1000);
        }
        ErrorProto errorResult = job.getStatus().getErrorResult();
        if (errorResult != null) {
            throw new Exception("Error running job: " + errorResult);
        }                   
    }
}

Answer 1

您缺少几个特定于appengine的pom设置。推荐的方法是从应用程序引擎原型创建pom.xml，如下所述：

mvn archetype:generate -Dappengine-version=1.9.30 -Dapplication-id=your-app-id -Dfilter=com.google.appengine.archetypes:appengine-skeleton-archetype

或者，您可以将构建插件添加到现有的pom.xml中，然后构建部分应该是这样的（这基本上是原型将为您创建的）：

<build>
  <!-- for hot reload of the web application-->
  <outputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</outputDirectory>
  <plugins>
    <plugin>
      <groupId>org.apache.maven.plugins</groupId>
      <version>3.1</version>
      <artifactId>maven-compiler-plugin</artifactId>
      <configuration>
        <source>1.7</source>
        <target>1.7</target>
      </configuration>
    </plugin>
    <plugin>
      <groupId>org.apache.maven.plugins</groupId>
      <artifactId>maven-war-plugin</artifactId>
      <version>2.4</version>
      <configuration>
        <archiveClasses>true</archiveClasses>
        <webResources>
          <!-- in order to interpolate version from pom into appengine-web.xml -->
          <resource>
            <directory>${basedir}/src/main/webapp/WEB-INF</directory>
            <filtering>true</filtering>
            <targetPath>WEB-INF</targetPath>
          </resource>
        </webResources>
      </configuration>
    </plugin>
    <plugin>
      <groupId>com.google.appengine</groupId>
      <artifactId>appengine-maven-plugin</artifactId>
      <version>${appengine.version}</version>
      <configuration>
        <enableJarClasses>false</enableJarClasses>
        <version>${app.version}</version>
        <!-- Comment in the below snippet to bind to all IPs instead of just localhost -->
        <address>0.0.0.0</address>
        <port>8080</port>
        <!-- Comment in the below snippet to enable local debugging with a remote debugger
                         like those included with Eclipse or IntelliJ -->
        <jvmFlags>
          <jvmFlag>-agentlib:jdwp=transport=dt_socket,address=8000,server=y,suspend=n</jvmFlag>
        </jvmFlags>
      </configuration>
    </plugin>
    <plugin>
      <groupId>com.google.appengine</groupId>
      <artifactId>gcloud-maven-plugin</artifactId>
      <version>${gcloud.plugin.version}</version>
      <configuration>
        <set_default>true</set_default>
      </configuration>
    </plugin>
  </plugins>
</build>

您还应该将appengine sdk添加到您的依赖项中，我的依赖项通常如下所示：

<!-- Compile/runtime dependencies -->
<dependency>
  <groupId>com.google.appengine</groupId>
  <artifactId>appengine-api-1.0-sdk</artifactId>
  <version>${appengine.version}</version>
</dependency>
<dependency>
  <groupId>javax.servlet</groupId>
  <artifactId>servlet-api</artifactId>
  <version>2.5</version>
  <scope>provided</scope>
</dependency>
<dependency>
  <groupId>jstl</groupId>
  <artifactId>jstl</artifactId>
  <version>1.2</version>
</dependency>

最后但同样重要的是，appengine项目的打包通常设置为WAR

<packaging>war</packaging>

设置好所有这些之后（并且在web-INF中有一个appengine-web.xml），您可以使用部署您的appengine应用程序

mvn appengine:update

我建议您创建一个具有原型的项目，并在新项目中复制您的内容。这比将所有这些配置添加到现有项目中要容易得多。