{"id":663,"date":"2019-02-26T09:08:48","date_gmt":"2019-02-26T00:08:48","guid":{"rendered":"https:\/\/www.kwonline.org\/memo2\/?p=663"},"modified":"2021-05-25T11:01:54","modified_gmt":"2021-05-25T02:01:54","slug":"spreadsheet-as-bq-ext-table","status":"publish","type":"post","link":"https:\/\/www.kwonline.org\/memo2\/2019\/02\/26\/spreadsheet-as-bq-ext-table\/","title":{"rendered":"Google Spreadsheet\u306e1\u679a\u76ee\u30b7\u30fc\u30c8\u3092BigQuery\u306e\u5916\u90e8\u30c6\u30fc\u30d6\u30eb\u306b\u3059\u308b"},"content":{"rendered":"<p>\t\t\t\t\u793e\u5185\u306e\u540c\u30c6\u30f3\u30d7\u30ec\u306e\u30b9\u30d7\u30ec\u30c3\u30c9\u30b7\u30fc\u30c8\u307e\u3068\u3081\u308b\u306e\u3053\u308c\u3067\u3084\u308b\u3068\u697d\u3060\u3063\u305f\u306e\u3067\u30e1\u30e2\u3002<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n#!\/bin\/bash\r\n\r\nSCHEMA=date:DATE,id:STRING,description:STRING\r\nURL=https:\/\/docs.google.com\/spreadsheets\/hoge\/fuga\r\nPRJ=myproject\r\nDS=mydataset\r\nTBL=mytable\r\n\r\nbq mk --external_table_definition=${SCHEMA}@GOOGLE_SHEETS=${URL} ${PRJ}:${DS}.${TBL}\r\n<\/pre>\n<p>\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306f\u5148\u306b\u4f5c\u3063\u3066\u304a\u304b\u306a\u3044\u3068\u6012\u3089\u308c\u308b\u3002<br \/>\n\u3057\u304b\u3057\u3053\u306e\u65b9\u6cd5\u306f\u30b7\u30fc\u30c81\u679a\u76ee\u3057\u304b\u5bfe\u5fdc\u3057\u3066\u306a\u3044\u4ed5\u69d8\u3002<\/p>\n<p>\u8907\u6570\u30b7\u30fc\u30c8\u3082\u6271\u3046\u306a\u3089AppsScript\u3067BQ\u306e\u30cd\u30a4\u30c6\u30a3\u30d6\u30c6\u30fc\u30d6\u30eb\u76f4\u63a5\u4f5c\u308b\u3057\u304b\u306a\u3044\u3002<br \/>\n\u5916\u90e8\u30c6\u30fc\u30d6\u30eb\u3058\u3083\u306a\u3044\u3051\u3069\u3055\u3002<\/p>\n<pre class=\"brush: jscript; title: ; notranslate\" title=\"\">\r\nfunction main() {\r\n  var projectId = 'myproject';\r\n  var datasetId = 'mydataset';\r\n\r\n  var app = SpreadsheetApp.getActiveSpreadsheet();\r\n  var sheet = app.getActiveSheet();\r\n  var tableId = sheet.getSheetName();\r\n  \r\n  var table = {\r\n    tableReference: {\r\n      projectId: projectId,\r\n      datasetId: datasetId,\r\n      tableId: tableId\r\n    },\r\n    schema: {\r\n      fields: &#x5B;\r\n        {name: 'date', type: 'date'},\r\n        {name: 'id', type: 'string'},\r\n        {name: 'description', type: 'string'}\r\n      ]\r\n    }\r\n  };\r\n  try{\r\n    BigQuery.Tables.remove(projectId, datasetId, tableId); \r\n  } catch(e) {}\r\n  table = BigQuery.Tables.insert(table, projectId, datasetId);\r\n  \r\n  var range = sheet.getDataRange();\r\n  var blob = Utilities.newBlob(convCsv(range)).setContentType('application\/octet-stream');\r\n  var job = {\r\n    configuration: {\r\n      load: {\r\n        destinationTable: {\r\n          projectId: projectId,\r\n          datasetId: datasetId,\r\n          tableId: tableId\r\n        },\r\n        skipLeadingRows: 1\r\n      }\r\n    }\r\n  };\r\n  job = BigQuery.Jobs.insert(job, projectId, blob);\r\n}\r\n\r\nfunction convCsv(range) {\r\n  try {\r\n    var data = range.getValues();\r\n    var ret = &quot;&quot;;\r\n    if (data.length &gt; 1) {\r\n      var csv = &quot;&quot;;\r\n      for (var i = 0; i &lt; data.length; i++) {\r\n        for (var j = 0; j &lt; data&#x5B;i].length; j++) {\r\n          if (data&#x5B;i]&#x5B;j].toString().indexOf(&quot;,&quot;) != -1) {\r\n            data&#x5B;i]&#x5B;j] = &quot;\\&quot;&quot; + data&#x5B;i]&#x5B;j] + &quot;\\&quot;&quot;;\r\n          }\r\n        }\r\n        if (i &lt; data.length-1) {\r\n          csv += data&#x5B;i].join(&quot;,&quot;) + &quot;\\r\\n&quot;;\r\n        } else {\r\n          csv += data&#x5B;i];\r\n        }\r\n      }\r\n      ret = csv;\r\n    }\r\n    return ret;\r\n  }\r\n  catch(e) {\r\n    Logger.log(e);\r\n  }\r\n}\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u793e\u5185\u306e\u540c\u30c6\u30f3\u30d7\u30ec\u306e\u30b9\u30d7\u30ec\u30c3\u30c9\u30b7\u30fc\u30c8\u307e\u3068\u3081\u308b\u306e\u3053\u308c\u3067\u3084\u308b\u3068\u697d\u3060\u3063\u305f\u306e\u3067\u30e1\u30e2\u3002 #!\/bin\/bash SCHEMA=date:DATE,id:STRING,description:STRING URL=https:\/\/do [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[21,5,7,8],"tags":[],"class_list":["post-663","post","type-post","status-publish","format-standard","hentry","category-data-engineering","category-google","category-javascript","category-linux"],"_links":{"self":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts\/663","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/comments?post=663"}],"version-history":[{"count":1,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts\/663\/revisions"}],"predecessor-version":[{"id":1461,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts\/663\/revisions\/1461"}],"wp:attachment":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/media?parent=663"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/categories?post=663"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/tags?post=663"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}