diff --git a/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/Caching-practice-lab-2.ipynb b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/Caching-practice-lab-2.ipynb new file mode 100644 index 000000000..965527367 --- /dev/null +++ b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/Caching-practice-lab-2.ipynb @@ -0,0 +1,213 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "e9ae4c8b-4599-4fbb-a545-76b6e3bcb84d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "== Physical Plan ==\n", + "AdaptiveSparkPlan isFinalPlan=false\n", + "+- ObjectHashAggregate(keys=[device_id#598, device_type#601], functions=[collect_list(user_id#568, 0, 0)])\n", + " +- ObjectHashAggregate(keys=[device_id#598, device_type#601], functions=[partial_collect_list(user_id#568, 0, 0)])\n", + " +- Project [device_id#598, device_type#601, user_id#568]\n", + " +- SortMergeJoin [device_id#598], [device_id#569], Inner\n", + " :- Sort [device_id#598 ASC NULLS FIRST], false, 0\n", + " : +- Exchange hashpartitioning(device_id#598, 4), ENSURE_REQUIREMENTS, [plan_id=735]\n", + " : +- Filter isnotnull(device_id#598)\n", + " : +- FileScan csv [device_id#598,device_type#601] Batched: false, DataFilters: [isnotnull(device_id#598)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/devices.csv], PartitionFilters: [], PushedFilters: [IsNotNull(device_id)], ReadSchema: struct\n", + " +- Sort [device_id#569 ASC NULLS FIRST], false, 0\n", + " +- Exchange hashpartitioning(device_id#569, 4), ENSURE_REQUIREMENTS, [plan_id=736]\n", + " +- Filter isnotnull(device_id#569)\n", + " +- InMemoryTableScan [user_id#568, device_id#569], [isnotnull(device_id#569)]\n", + " +- InMemoryRelation [user_id#568, device_id#569, event_counts#606L, host_array#607], StorageLevel(disk, memory, deserialized, 1 replicas)\n", + " +- AdaptiveSparkPlan isFinalPlan=false\n", + " +- ObjectHashAggregate(keys=[user_id#17, device_id#18], functions=[count(1), collect_list(distinct host#20, 0, 0)])\n", + " +- Exchange hashpartitioning(user_id#17, device_id#18, 4), ENSURE_REQUIREMENTS, [plan_id=752]\n", + " +- ObjectHashAggregate(keys=[user_id#17, device_id#18], functions=[merge_count(1), partial_collect_list(distinct host#20, 0, 0)])\n", + " +- HashAggregate(keys=[user_id#17, device_id#18, host#20], functions=[merge_count(1)])\n", + " +- Exchange hashpartitioning(user_id#17, device_id#18, host#20, 4), ENSURE_REQUIREMENTS, [plan_id=748]\n", + " +- HashAggregate(keys=[user_id#17, device_id#18, host#20], functions=[partial_count(1)])\n", + " +- Filter isnotnull(user_id#17)\n", + " +- FileScan csv [user_id#17,device_id#18,host#20] Batched: false, DataFilters: [isnotnull(user_id#17)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/events.csv], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct\n", + "\n", + "\n", + "== Physical Plan ==\n", + "AdaptiveSparkPlan isFinalPlan=false\n", + "+- ObjectHashAggregate(keys=[user_id#568], functions=[max(event_counts#606L), collect_list(device_id#569, 0, 0)])\n", + " +- ObjectHashAggregate(keys=[user_id#568], functions=[partial_max(event_counts#606L), partial_collect_list(device_id#569, 0, 0)])\n", + " +- Project [user_id#568, device_id#569, event_counts#606L]\n", + " +- SortMergeJoin [user_id#568], [user_id#614], Inner\n", + " :- Sort [user_id#568 ASC NULLS FIRST], false, 0\n", + " : +- Exchange hashpartitioning(user_id#568, 4), ENSURE_REQUIREMENTS, [plan_id=788]\n", + " : +- Filter isnotnull(user_id#568)\n", + " : +- FileScan csv [user_id#568,device_id#569] Batched: false, DataFilters: [isnotnull(user_id#568)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/events.csv], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct\n", + " +- Sort [user_id#614 ASC NULLS FIRST], false, 0\n", + " +- Exchange hashpartitioning(user_id#614, 4), ENSURE_REQUIREMENTS, [plan_id=789]\n", + " +- Filter isnotnull(user_id#614)\n", + " +- InMemoryTableScan [user_id#614, event_counts#606L], [isnotnull(user_id#614)]\n", + " +- InMemoryRelation [user_id#614, device_id#615, event_counts#606L, host_array#607], StorageLevel(disk, memory, deserialized, 1 replicas)\n", + " +- AdaptiveSparkPlan isFinalPlan=false\n", + " +- ObjectHashAggregate(keys=[user_id#17, device_id#18], functions=[count(1), collect_list(distinct host#20, 0, 0)])\n", + " +- Exchange hashpartitioning(user_id#17, device_id#18, 4), ENSURE_REQUIREMENTS, [plan_id=805]\n", + " +- ObjectHashAggregate(keys=[user_id#17, device_id#18], functions=[merge_count(1), partial_collect_list(distinct host#20, 0, 0)])\n", + " +- HashAggregate(keys=[user_id#17, device_id#18, host#20], functions=[merge_count(1)])\n", + " +- Exchange hashpartitioning(user_id#17, device_id#18, host#20, 4), ENSURE_REQUIREMENTS, [plan_id=801]\n", + " +- HashAggregate(keys=[user_id#17, device_id#18, host#20], functions=[partial_count(1)])\n", + " +- Filter isnotnull(user_id#17)\n", + " +- FileScan csv [user_id#17,device_id#18,host#20] Batched: false, DataFilters: [isnotnull(user_id#17)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/events.csv], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct\n", + "\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "import org.apache.spark.sql.functions._\n", + "import org.apache.spark.storage.StorageLevel\n", + "users: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [user_id: int, device_id: int ... 4 more fields]\n", + "devices: org.apache.spark.sql.DataFrame = [device_id: int, browser_type: string ... 2 more fields]\n", + "executionDate: String = 2023-01-01\n", + "eventsAggregated: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [user_id: int, device_id: int ... 2 more fields]\n", + "usersAndDevices: org.apache.spark.sql.DataFrame = [user_id: int, user_id: int ... 2 more fields]\n", + "devicesOnEvents: org.apache.spark.sql.DataFrame = [device_id: int, device_type: string ... 3 more fields]\n", + "res1: Array[org.apache.spark.sql.Row] = Array([-2147470439,-2147470439,3,WrappedArray(378988111, 378988111, 378988111)])\n" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import org.apache.spark.sql.functions._\n", + "import org.apache.spark.storage.StorageLevel\n", + "\n", + "spark.conf.set(\"spark.sql.autoBroadcastJoinThreshold\", \"-1\")\n", + "spark.conf.set(\"spark.sql.analyzer.failAmbiguousSelfJoin\", \"false\")\n", + "spark.conf.set(\"spark.sql.shuffle.partitions\", \"4\")\n", + "\n", + "val users = spark.read.option(\"header\", \"true\")\n", + " .option(\"inferSchema\", \"true\")\n", + " .csv(\"/home/iceberg/data/events.csv\")\n", + " .where($\"user_id\".isNotNull)\n", + "\n", + "users.createOrReplaceTempView(\"events\")\n", + "\n", + "val devices = spark.read.option(\"header\", \"true\")\n", + " .option(\"inferSchema\", \"true\")\n", + " .csv(\"/home/iceberg/data/devices.csv\")\n", + "\n", + "devices.createOrReplaceTempView(\"devices\")\n", + "\n", + "val executionDate = \"2023-01-01\"\n", + "\n", + "//Caching here should be < 5 GBs or used for broadcast join\n", + "//You need to tune executor memory otherwise it'll spill to disk and be slow\n", + "//Don't really try using any of the other StorageLevel besides MEMORY_ONLY\n", + "\n", + "val eventsAggregated = spark.sql(f\"\"\"\n", + " SELECT user_id, \n", + " device_id, \n", + " COUNT(1) as event_counts, \n", + " COLLECT_LIST(DISTINCT host) as host_array\n", + " FROM events\n", + " GROUP BY 1,2\n", + "\"\"\").cache()\n", + "\n", + "// eventsAggregated.write.mode(\"overwrite\").saveAsTable(\"bootcamp.events_aggregated_staging\")\n", + "\n", + "spark.sql(f\"\"\"\n", + " CREATE TABLE IF NOT EXISTS bootcamp.events_aggregated_staging (\n", + " user_id BIGINT,\n", + " device_id BIGINT,\n", + " event_counts BIGINT,\n", + " host_array ARRAY\n", + " )\n", + " PARTITIONED BY (ds STRING)\n", + "\"\"\")\n", + "\n", + "\n", + "val usersAndDevices = users\n", + " .join(eventsAggregated, eventsAggregated(\"user_id\") === users(\"user_id\"))\n", + " .groupBy(users(\"user_id\"))\n", + " .agg(\n", + " users(\"user_id\"),\n", + " max(eventsAggregated(\"event_counts\")).as(\"total_hits\"),\n", + " collect_list(eventsAggregated(\"device_id\")).as(\"devices\")\n", + " )\n", + "\n", + "val devicesOnEvents = devices\n", + " .join(eventsAggregated, devices(\"device_id\") === eventsAggregated(\"device_id\"))\n", + " .groupBy(devices(\"device_id\"), devices(\"device_type\"))\n", + " .agg(\n", + " devices(\"device_id\"),\n", + " devices(\"device_type\"),\n", + " collect_list(eventsAggregated(\"user_id\")).as(\"users\")\n", + " )\n", + "\n", + "devicesOnEvents.explain()\n", + "usersAndDevices.explain()\n", + "\n", + "devicesOnEvents.take(1)\n", + "usersAndDevices.take(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "c36feefd-a8cd-4749-81fd-8f7883349ba1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "res18: eventsAggregated.type = [user_id: int, device_id: int ... 2 more fields]\n" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eventsAggregated.unpersist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02886a49-4dc5-4e00-9cad-20926166f388", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spylon-kernel", + "language": "scala", + "name": "spylon-kernel" + }, + "language_info": { + "codemirror_mode": "text/x-scala", + "file_extension": ".scala", + "help_links": [ + { + "text": "MetaKernel Magics", + "url": "https://metakernel.readthedocs.io/en/latest/source/README.html" + } + ], + "mimetype": "text/x-scala", + "name": "scala", + "pygments_lexer": "scala", + "version": "0.4.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/DatasetApi-practice-lab-2.ipynb b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/DatasetApi-practice-lab-2.ipynb new file mode 100644 index 000000000..125aca5a5 --- /dev/null +++ b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/DatasetApi-practice-lab-2.ipynb @@ -0,0 +1,371 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "22b842be-6a82-4127-b937-ead4103a92e8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Intitializing Scala interpreter ..." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Spark Web UI available at http://120fa5ac9a2d:4041\n", + "SparkContext available as 'sc' (version = 3.5.1, master = local[*], app id = local-1734316680820)\n", + "SparkSession available as 'spark'\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "res0: org.apache.spark.sql.SparkSession = org.apache.spark.sql.SparkSession@442b0b59\n" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spark" + ] + }, + { + "cell_type": "markdown", + "id": "2295da8f-3623-43ba-af46-6fe19ec9ffca", + "metadata": {}, + "source": [ + "# If something is nullabe, you need to wrap the value type in Option[] - this helps enforce assumptions about the pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "73b5384f-be28-49e3-9bcf-4b9783ba7d91", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "import org.apache.spark.sql.SparkSession\n", + "sparkSession: org.apache.spark.sql.SparkSession = org.apache.spark.sql.SparkSession@442b0b59\n", + "defined class Event\n" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import org.apache.spark.sql.SparkSession \n", + "\n", + "val sparkSession = SparkSession.builder.appName(\"Juptyer\").getOrCreate()\n", + "\n", + "//TODO Illustrate how this fails if you change from Option[String] to String for referrer\n", + "case class Event (\n", + " //Option is a way to handle NULL more gracefully\n", + " user_id: Option[Integer],\n", + " device_id: Option[Integer],\n", + " referrer: Option[String],\n", + " host: String,\n", + " url: String,\n", + " event_time: String\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "25c3320b-03ab-4969-88a7-f7771534d70c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dummyData: List[Event] = List(Event(Some(1),Some(2),Some(linkedin),eczachly.com,/signup,2023-01-01), Event(Some(3),Some(7),Some(twitter),eczachly.com,/signup,2023-01-01))\n" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "val dummyData = List(\n", + " Event(user_id=Some(1), device_id=Some(2), referrer=Some(\"linkedin\"), host=\"eczachly.com\", url=\"/signup\", event_time=\"2023-01-01\"),\n", + " Event(user_id=Some(3), device_id=Some(7), referrer=Some(\"twitter\"), host=\"eczachly.com\", url=\"/signup\", event_time=\"2023-01-01\")\n", + " )\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f7190f95-33b1-4779-8f0d-40e9046e8157", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "defined class Device\n", + "defined class EventWithDeviceInfo\n" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "//TODO Illustrate how this fails if you change from Option[Long] to Long\n", + "case class Device (\n", + " device_id: Integer,\n", + " browser_type: String,\n", + " os_type: String,\n", + " device_type: String\n", + ")\n", + "\n", + "case class EventWithDeviceInfo (\n", + " user_id: Integer,\n", + " device_id: Integer,\n", + " browser_type: String,\n", + " os_type: String,\n", + " device_type: String,\n", + " referrer: String,\n", + " host: String,\n", + " url: String,\n", + " event_time: String\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d045ae06-0f21-42f1-a271-f455e95b0879", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "import org.apache.spark.sql.Dataset\n", + "import sparkSession.implicits._\n", + "events: org.apache.spark.sql.Dataset[Event] = [user_id: int, device_id: int ... 4 more fields]\n", + "devices: org.apache.spark.sql.Dataset[Device] = [device_id: int, browser_type: string ... 2 more fields]\n", + "filteredViaDataset: org.apache.spark.sql.Dataset[Event] = [user_id: int, device_id: int ... 4 more fields]\n", + "filteredViaDataFrame: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [user_id: int, device_id: int ... 4 more fields]\n", + "filteredViaSparkSql: org.apache.spark.sql.DataFrame = [user_id: int, device_id: int ... 4 more fields]\n" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "import org.apache.spark.sql.Dataset\n", + "\n", + "import sparkSession.implicits._\n", + "\n", + "// Applying this case class before hand is very powerful, enforces Nullability/non-nullability at runtime!\n", + "val events: Dataset[Event] = sparkSession.read.option(\"header\", \"true\")\n", + " .option(\"inferSchema\", \"true\")\n", + " .csv(\"/home/iceberg/data/events.csv\")\n", + " .as[Event]\n", + "\n", + "val devices: Dataset[Device] = sparkSession.read.option(\"header\", \"true\")\n", + " .option(\"inferSchema\", \"true\")\n", + " .csv(\"/home/iceberg/data/devices.csv\")\n", + " .as[Device]\n", + "\n", + "devices.createOrReplaceTempView(\"devices\")\n", + "events.createOrReplaceTempView(\"events\")\n", + "\n", + "// For simple transformations, you can see that these approaches are very similar. Dataset is winning slightly because of the quality enforcement\n", + "val filteredViaDataset = events.filter(event => event.user_id.isDefined && event.device_id.isDefined)\n", + "val filteredViaDataFrame = events.toDF().where($\"user_id\".isNotNull && $\"device_id\".isNotNull)\n", + "val filteredViaSparkSql = sparkSession.sql(\"SELECT * FROM events WHERE user_id IS NOT NULL AND device_id IS NOT NULL\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "80517409-a26e-4aa3-b383-56b928f0c9d3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "combinedViaDatasets: org.apache.spark.sql.Dataset[EventWithDeviceInfo] = [user_id: int, device_id: int ... 7 more fields]\n" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "// This will fail if user_id is None\n", + "val combinedViaDatasets = filteredViaDataset\n", + " .joinWith(devices, events(\"device_id\") === devices(\"device_id\"), \"inner\")\n", + " .map{ case (event: Event, device: Device) => EventWithDeviceInfo(\n", + " user_id=event.user_id.get,\n", + " device_id=device.device_id,\n", + " browser_type=device.browser_type,\n", + " os_type=device.os_type,\n", + " device_type=device.device_type,\n", + " referrer=event.referrer.getOrElse(\"unknow\"),\n", + " host=event.host,\n", + " url=event.url,\n", + " event_time=event.event_time\n", + " ) }\n", + " .map { eventWithDevice =>\n", + " // Convert browser_type to uppercase while maintaining immutability\n", + " eventWithDevice.copy(browser_type = eventWithDevice.browser_type.toUpperCase)\n", + " }\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3ce150e3-e5b7-4ece-8803-8189762625ea", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "EventWithDeviceInfo(1037710827,532630305,OTHER,Other,Other,unknow,www.zachwilson.tech,/,2021-03-08 17:27:24.241)\n", + "EventWithDeviceInfo(925588856,532630305,OTHER,Other,Other,unknow,www.eczachly.com,/,2021-05-10 11:26:21.247)\n", + "EventWithDeviceInfo(-1180485268,532630305,OTHER,Other,Other,unknow,admin.zachwilson.tech,/,2021-02-17 16:19:30.738)\n", + "EventWithDeviceInfo(-1044833855,532630305,OTHER,Other,Other,unknow,www.zachwilson.tech,/,2021-09-24 15:53:14.466)\n", + "EventWithDeviceInfo(747494706,532630305,OTHER,Other,Other,unknow,www.zachwilson.tech,/,2021-09-26 16:03:17.535)\n", + "+-----------+---------+------------+-------+-----------+--------+--------------------+---+--------------------+\n", + "| user_id|device_id|browser_type|os_type|device_type|referrer| host|url| event_time|\n", + "+-----------+---------+------------+-------+-----------+--------+--------------------+---+--------------------+\n", + "| 1037710827|532630305| Other| Other| Other| NULL| www.zachwilson.tech| /|2021-03-08 17:27:...|\n", + "| 925588856|532630305| Other| Other| Other| NULL| www.eczachly.com| /|2021-05-10 11:26:...|\n", + "|-1180485268|532630305| Other| Other| Other| NULL|admin.zachwilson....| /|2021-02-17 16:19:...|\n", + "|-1044833855|532630305| Other| Other| Other| NULL| www.zachwilson.tech| /|2021-09-24 15:53:...|\n", + "| 747494706|532630305| Other| Other| Other| NULL| www.zachwilson.tech| /|2021-09-26 16:03:...|\n", + "+-----------+---------+------------+-------+-----------+--------+--------------------+---+--------------------+\n", + "only showing top 5 rows\n", + "\n", + "+-----------+---------+------------+-------+-----------+--------+--------------------+---+--------------------+\n", + "| user_id|device_id|browser_type|os_type|device_type|referrer| host|url| event_time|\n", + "+-----------+---------+------------+-------+-----------+--------+--------------------+---+--------------------+\n", + "| 1037710827|532630305| Other| Other| Other| NULL| www.zachwilson.tech| /|2021-03-08 17:27:...|\n", + "| 925588856|532630305| Other| Other| Other| NULL| www.eczachly.com| /|2021-05-10 11:26:...|\n", + "|-1180485268|532630305| Other| Other| Other| NULL|admin.zachwilson....| /|2021-02-17 16:19:...|\n", + "|-1044833855|532630305| Other| Other| Other| NULL| www.zachwilson.tech| /|2021-09-24 15:53:...|\n", + "| 747494706|532630305| Other| Other| Other| NULL| www.zachwilson.tech| /|2021-09-26 16:03:...|\n", + "+-----------+---------+------------+-------+-----------+--------+--------------------+---+--------------------+\n", + "only showing top 5 rows\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "combinedViaDataFrames: org.apache.spark.sql.DataFrame = [user_id: int, device_id: int ... 7 more fields]\n", + "combinedViaSparkSQL: org.apache.spark.sql.DataFrame = [user_id: int, device_id: int ... 7 more fields]\n", + "rows: Array[EventWithDeviceInfo] = Array(EventWithDeviceInfo(1037710827,532630305,OTHER,Other,Other,unknow,www.zachwilson.tech,/,2021-03-08 17:27:24.241), EventWithDeviceInfo(925588856,532630305,OTHER,Other,Other,unknow,www.eczachly.com,/,2021-05-10 11:26:21.247), EventWithDeviceInfo(-1180485268,532630305,OTHER,Other,Other,unknow,admin.zachwilson.tech,/,2021-02-17 16:19:30.738), EventWithDeviceInfo(-1044833855,532630305,OTHER,Other,Other,unknow,www.zachwilson.tech,/,2021-09-24 15:53:14.466), EventWithDeviceInfo(747494706,532630305,OTHER,Other,Other,unknow,www.zachwilson.tech,/,2021-...\n" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "// DataFrames give up some of the intellisense because you no longer have static typing\n", + "val combinedViaDataFrames = filteredViaDataFrame.as(\"e\")\n", + " //Make sure to use triple equals when using data frames\n", + " .join(devices.as(\"d\"), $\"e.device_id\" === $\"d.device_id\", \"inner\")\n", + " .select(\n", + " $\"e.user_id\",\n", + " $\"d.device_id\",\n", + " $\"d.browser_type\",\n", + " $\"d.os_type\",\n", + " $\"d.device_type\",\n", + " $\"e.referrer\",\n", + " $\"e.host\",\n", + " $\"e.url\",\n", + " $\"e.event_time\"\n", + " )\n", + "\n", + "//Creating temp views is a good strategy if you're leveraging SparkSQL\n", + "filteredViaSparkSql.createOrReplaceTempView(\"filtered_events\")\n", + "val combinedViaSparkSQL = spark.sql(f\"\"\"\n", + " SELECT \n", + " fe.user_id,\n", + " d.device_id,\n", + " d.browser_type,\n", + " d.os_type,\n", + " d.device_type,\n", + " fe. referrer,\n", + " fe.host,\n", + " fe.url,\n", + " fe.event_time\n", + " FROM filtered_events fe \n", + " JOIN devices d ON fe.device_id = d.device_id\n", + "\"\"\")\n", + "\n", + "\n", + "val rows= combinedViaDatasets.take(5)\n", + "rows.foreach(println)\n", + "combinedViaDataFrames.show(5)\n", + "combinedViaSparkSQL.show(5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1db45765-b3d1-4059-92f1-e0414f502eb7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spylon-kernel", + "language": "scala", + "name": "spylon-kernel" + }, + "language_info": { + "codemirror_mode": "text/x-scala", + "file_extension": ".scala", + "help_links": [ + { + "text": "MetaKernel Magics", + "url": "https://metakernel.readthedocs.io/en/latest/source/README.html" + } + ], + "mimetype": "text/x-scala", + "name": "scala", + "pygments_lexer": "scala", + "version": "0.4.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/bucket-joins-in-iceberg-practice-lab-2.ipynb b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/bucket-joins-in-iceberg-practice-lab-2.ipynb new file mode 100644 index 000000000..968c4b5bd --- /dev/null +++ b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/bucket-joins-in-iceberg-practice-lab-2.ipynb @@ -0,0 +1,141 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "374411c5-a48f-4739-9031-d638638633a3", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "// In python use: from pyspark.sql.functions import broadcast, split, lit\n", + "import org.apache.spark.sql.functions.{broadcast, split, lit}\n", + "\n", + "\n", + "val matchesBucketed = spark.read.option(\"header\", \"true\")\n", + " .option(\"inferSchema\", \"true\")\n", + " .csv(\"/home/iceberg/data/matches.csv\")\n", + "val matchDetailsBucketed = spark.read.option(\"header\", \"true\")\n", + " .option(\"inferSchema\", \"true\")\n", + " .csv(\"/home/iceberg/data/match_details.csv\")\n", + "\n", + "\n", + "spark.sql(\"\"\"DROP TABLE IF EXISTS bootcamp.matches_bucketed\"\"\")\n", + "val bucketedDDL = \"\"\"\n", + "CREATE TABLE IF NOT EXISTS bootcamp.matches_bucketed (\n", + " match_id STRING,\n", + " is_team_game BOOLEAN,\n", + " playlist_id STRING,\n", + " completion_date TIMESTAMP\n", + " )\n", + " USING iceberg\n", + " PARTITIONED BY (completion_date, bucket(16, match_id));\n", + " \"\"\"\n", + " spark.sql(bucketedDDL)\n", + "\n", + "// matchesBucketed.select(\n", + "// $\"match_id\", $\"is_team_game\", $\"playlist_id\", $\"completion_date\"\n", + "// )\n", + "// .write.mode(\"append\")\n", + "// .partitionBy(\"completion_date\")\n", + "// .bucketBy(16, \"match_id\").saveAsTable(\"bootcamp.matches_bucketed\")\n", + "\n", + "\n", + "// val bucketedDetailsDDL = \"\"\"\n", + "// CREATE TABLE IF NOT EXISTS bootcamp.match_details_bucketed (\n", + "// match_id STRING,\n", + "// player_gamertag STRING,\n", + "// player_total_kills INTEGER,\n", + "// player_total_deaths INTEGER\n", + "// )\n", + "// USING iceberg\n", + "// PARTITIONED BY (bucket(16, match_id));\n", + "// \"\"\"\n", + "// spark.sql(bucketedDetailsDDL)\n", + "\n", + "// matchDetailsBucketed.select(\n", + "// $\"match_id\", $\"player_gamertag\", $\"player_total_kills\", $\"player_total_deaths\")\n", + "// .write.mode(\"append\")\n", + "// .bucketBy(16, \"match_id\").saveAsTable(\"bootcamp.match_details_bucketed\")\n", + "\n", + "spark.conf.set(\"spark.sql.autoBroadcastJoinThreshold\", \"-1\")\n", + "\n", + "matchesBucketed.createOrReplaceTempView(\"matches\")\n", + "matchDetailsBucketed.createOrReplaceTempView(\"match_details\")\n", + "\n", + "spark.sql(\"\"\"\n", + " SELECT * FROM bootcamp.match_details_bucketed mdb JOIN bootcamp.matches_bucketed md \n", + " ON mdb.match_id = md.match_id\n", + " AND md.completion_date = DATE('2016-01-01')\n", + " \n", + "\"\"\").explain()\n", + "\n", + "\n", + "spark.sql(\"\"\"\n", + " SELECT * FROM match_details mdb JOIN matches md ON mdb.match_id = md.match_id\n", + " \n", + "\"\"\").explain()\n", + "\n", + "// spark.conf.set(\"spark.sql.autoBroadcastJoinThreshold\", \"1000000000000\")\n", + "\n", + "// val broadcastFromThreshold = matches.as(\"m\").join(matchDetails.as(\"md\"), $\"m.match_id\" === $\"md.match_id\")\n", + "// .select($\"m.completion_date\", $\"md.player_gamertag\", $\"md.player_total_kills\")\n", + "// .take(5)\n", + "\n", + "// val explicitBroadcast = matches.as(\"m\").join(broadcast(matchDetails).as(\"md\"), $\"m.match_id\" === $\"md.match_id\")\n", + "// .select($\"md.*\", split($\"completion_date\", \" \").getItem(0).as(\"ds\"))\n", + "\n", + "// val bucketedValues = matchDetailsBucketed.as(\"mdb\").join(matchesBucketed.as(\"mb\"), $\"mb.match_id\" === $\"mdb.match_id\").explain()\n", + "// // .take(5)\n", + "\n", + "// val values = matchDetailsBucketed.as(\"m\").join(matchesBucketed.as(\"md\"), $\"m.match_id\" === $\"md.match_id\").explain()\n", + "\n", + "// explicitBroadcast.write.mode(\"overwrite\").insertInto(\"match_details_bucketed\")\n", + "\n", + "// matches.withColumn(\"ds\", split($\"completion_date\", \" \").getItem(0)).write.mode(\"overwrite\").insertInto(\"matches_bucketed\")\n", + "\n", + "// spark.sql(bucketedSQL)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc8adb02-d5bd-4e84-a671-48991772d233", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f1eecb6-ca9a-4b5c-b046-b3a0dd1ff3bf", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spylon-kernel", + "language": "scala", + "name": "spylon-kernel" + }, + "language_info": { + "codemirror_mode": "text/x-scala", + "file_extension": ".scala", + "help_links": [ + { + "text": "MetaKernel Magics", + "url": "https://metakernel.readthedocs.io/en/latest/source/README.html" + } + ], + "mimetype": "text/x-scala", + "name": "scala", + "pygments_lexer": "scala", + "version": "0.4.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/event_data_pyspark-practice-lab-1.ipynb b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/event_data_pyspark-practice-lab-1.ipynb new file mode 100644 index 000000000..fb330c821 --- /dev/null +++ b/intermediate-bootcamp/materials/3-spark-fundamentals/notebooks/lecture-lab-hands-on/event_data_pyspark-practice-lab-1.ipynb @@ -0,0 +1,2239 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "81cca085-dba2-42eb-a13b-fa64b6e86583", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------+----------+--------+--------------------+----------+--------------------+-------------------+\n", + "| user_id| device_id|referrer| host| url| event_time| event_date|\n", + "+-----------+----------+--------+--------------------+----------+--------------------+-------------------+\n", + "| 1037710827| 532630305| NULL| www.zachwilson.tech| /|2021-03-08 17:27:...|2021-03-08 00:00:00|\n", + "| 925588856| 532630305| NULL| www.eczachly.com| /|2021-05-10 11:26:...|2021-05-10 00:00:00|\n", + "|-1180485268| 532630305| NULL|admin.zachwilson....| /|2021-02-17 16:19:...|2021-02-17 00:00:00|\n", + "|-1044833855| 532630305| NULL| www.zachwilson.tech| /|2021-09-24 15:53:...|2021-09-24 00:00:00|\n", + "| 747494706| 532630305| NULL| www.zachwilson.tech| /|2021-09-26 16:03:...|2021-09-26 00:00:00|\n", + "| 747494706| 532630305| NULL|admin.zachwilson....| /|2021-02-21 16:08:...|2021-02-21 00:00:00|\n", + "| -824540328| 532630305| NULL|admin.zachwilson....| /|2021-09-28 17:23:...|2021-09-28 00:00:00|\n", + "| -824540328| 532630305| NULL| www.eczachly.com| /|2021-09-29 01:22:...|2021-09-29 00:00:00|\n", + "| 1833036683| 532630305| NULL|admin.zachwilson....| /|2021-01-24 03:15:...|2021-01-24 00:00:00|\n", + "|-2134824313| 532630305| NULL| www.eczachly.com| /|2021-01-25 00:03:...|2021-01-25 00:00:00|\n", + "|-1809929467|-906264142| NULL|admin.zachwilson....|/.git/HEAD|2021-02-22 01:36:...|2021-02-22 00:00:00|\n", + "| 2002285749|-906264142| NULL| www.eczachly.com| /|2021-02-22 02:25:...|2021-02-22 00:00:00|\n", + "|-1562965412| 532630305| NULL| www.zachwilson.tech| /|2021-01-30 20:46:...|2021-01-30 00:00:00|\n", + "|-1099860451| 532630305| NULL| www.eczachly.com| /|2021-02-04 23:49:...|2021-02-04 00:00:00|\n", + "| 1246896869|-906264142| NULL| www.zachwilson.tech| /|2021-02-22 02:50:...|2021-02-22 00:00:00|\n", + "| -629331502|-906264142| NULL|admin.zachwilson....|/.git/HEAD|2021-02-22 23:51:...|2021-02-22 00:00:00|\n", + "|-1913422462|-906264142| NULL| www.eczachly.com| /|2021-02-23 00:17:...|2021-02-23 00:00:00|\n", + "| 50429624| 532630305| NULL| www.eczachly.com| /|2022-12-28 01:38:...|2022-12-28 00:00:00|\n", + "| 222389292| 532630305| NULL| www.zachwilson.tech| /|2022-12-28 05:23:...|2022-12-28 00:00:00|\n", + "| -779924777| 532630305| NULL| www.zachwilson.tech| /|2022-12-28 16:45:...|2022-12-28 00:00:00|\n", + "+-----------+----------+--------+--------------------+----------+--------------------+-------------------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import expr, col, lit\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "# not required\n", + "# spark\n", + "\n", + "df = spark.read.option(\"header\", \"true\")\\\n", + " .csv(\"/home/iceberg/data/events.csv\")\\\n", + " .withColumn(\"event_date\", expr(\"DATE_TRUNC('day', event_time)\"))\n", + "\n", + "df.show()\n", + "# dont evert run collect() on entire dataset as it will OOM and crash kernel, unless data is aggregated\n", + "# df.collect()\n", + "\n", + "# this will oom and crash kernel\n", + "# df.join(df, lit(1) == lit(1)).collect()\n", + "\n", + "# this ran as there was limited data\n", + "# df.join(df, lit(1) == lit(1)).take(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "dce068df-3e21-429a-8716-abdd13e9406c", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------+-----------+--------------------+--------------------+--------------------+--------------------+-------------------+\n", + "| user_id| device_id| referrer| host| url| event_time| event_date|\n", + "+-----------+-----------+--------------------+--------------------+--------------------+--------------------+-------------------+\n", + "| 1129583063| 532630305| NULL|admin.zachwilson....| /|2021-01-07 09:21:...|2021-01-07 00:00:00|\n", + "| -648945006| 1088283544| NULL| www.eczachly.com| /|2021-01-07 02:58:...|2021-01-07 00:00:00|\n", + "|-1871780024| -158310583| NULL| www.eczachly.com| /|2021-01-07 04:17:...|2021-01-07 00:00:00|\n", + "| 203689086| 1088283544| NULL| www.eczachly.com|/blog/what-exactl...|2021-01-07 10:03:...|2021-01-07 00:00:00|\n", + "|-1180485268| 532630305| NULL| www.eczachly.com| /|2021-01-07 18:45:...|2021-01-07 00:00:00|\n", + "| 1129583063| 532630305| NULL| www.eczachly.com| /|2021-01-07 21:57:...|2021-01-07 00:00:00|\n", + "|-1381834161| -158310583| NULL| www.eczachly.com| /|2021-01-07 23:07:...|2021-01-07 00:00:00|\n", + "|-1373330946| 532630305| NULL| www.zachwilson.tech| /api/v1/spark-post|2021-01-07 18:53:...|2021-01-07 00:00:00|\n", + "|-1180485268| 532630305| NULL| www.zachwilson.tech| /|2021-01-07 19:20:...|2021-01-07 00:00:00|\n", + "|-1617088793| 1957784035|https://www.zachw...| www.zachwilson.tech| /contact|2021-01-07 00:19:...|2021-01-07 00:00:00|\n", + "|-1617088793| 1957784035|https://www.zachw...| www.zachwilson.tech| /api/v1/contact|2021-01-07 00:19:...|2021-01-07 00:00:00|\n", + "|-1617088793| 1957784035|https://www.zachw...| www.zachwilson.tech| /contact|2021-01-07 00:19:...|2021-01-07 00:00:00|\n", + "| -267976675| 1141939293|https://www.zachw...| www.zachwilson.tech| /contact|2021-01-07 01:57:...|2021-01-07 00:00:00|\n", + "| -267976675| 1141939293|https://www.zachw...| www.zachwilson.tech| /api/v1/contact|2021-01-07 01:57:...|2021-01-07 00:00:00|\n", + "| -267976675| 1141939293|https://www.zachw...| www.zachwilson.tech| /contact|2021-01-07 01:57:...|2021-01-07 00:00:00|\n", + "| 604069040| 1800580819| NULL| www.zachwilson.tech| /|2021-01-07 02:24:...|2021-01-07 00:00:00|\n", + "|-2135898320| 2066336516|http://www.laurel...| www.zachwilson.tech| /|2021-01-07 04:57:...|2021-01-07 00:00:00|\n", + "| -43429336|-1217993711| NULL| www.zachwilson.tech| /contact|2021-01-07 04:57:...|2021-01-07 00:00:00|\n", + "| 1198758947|-1307168836| NULL| www.zachwilson.tech| /blog|2021-01-07 09:57:...|2021-01-07 00:00:00|\n", + "|-1894709267| 1141939293|https://www.zachw...| www.zachwilson.tech| /|2021-01-07 10:12:...|2021-01-07 00:00:00|\n", + "+-----------+-----------+--------------------+--------------------+--------------------+--------------------+-------------------+\n", + "only showing top 20 rows\n", + "\n", + "+-----------+-----------+--------------------+--------------------+--------------------+--------------------+-------------------+\n", + "| user_id| device_id| referrer| host| url| event_time| event_date|\n", + "+-----------+-----------+--------------------+--------------------+--------------------+--------------------+-------------------+\n", + "| 1272828233| -643696601| NULL|admin.zachwilson....| /|2021-01-02 13:53:...|2021-01-02 00:00:00|\n", + "| 747494706| 532630305| NULL|admin.zachwilson....| /|2021-01-02 19:36:...|2021-01-02 00:00:00|\n", + "| 2110046626| 898871897| NULL|admin.zachwilson....| /wp-login.php|2021-01-02 19:57:...|2021-01-02 00:00:00|\n", + "| 1272828233| -643696601| NULL|admin.zachwilson....| /|2021-01-02 21:05:...|2021-01-02 00:00:00|\n", + "| 1272828233| -643696601| NULL|admin.zachwilson....| /|2021-01-02 21:37:...|2021-01-02 00:00:00|\n", + "| 1399665425|-2012543895| NULL| www.eczachly.com| /|2021-01-02 00:20:...|2021-01-02 00:00:00|\n", + "| 125243313| -290659081| NULL| www.eczachly.com| /|2021-01-02 02:06:...|2021-01-02 00:00:00|\n", + "| 632739597| -290659081| NULL| www.eczachly.com|/blog/what-exactl...|2021-01-02 02:58:...|2021-01-02 00:00:00|\n", + "|-1780827820| -290659081| NULL| www.eczachly.com| /|2021-01-02 04:45:...|2021-01-02 00:00:00|\n", + "| 632739597| -290659081| NULL| www.eczachly.com| /|2021-01-02 05:14:...|2021-01-02 00:00:00|\n", + "| 1047962242| -158310583| NULL| www.eczachly.com| /|2021-01-02 11:40:...|2021-01-02 00:00:00|\n", + "| 273700037| -290659081| NULL| www.eczachly.com| /|2021-01-02 07:51:...|2021-01-02 00:00:00|\n", + "| 1272828233| -643696601| NULL| www.eczachly.com| /|2021-01-02 08:14:...|2021-01-02 00:00:00|\n", + "| 210988258| 1088283544| NULL| www.eczachly.com| /contact|2021-01-02 11:11:...|2021-01-02 00:00:00|\n", + "| 273700037| -290659081| NULL| www.eczachly.com| /|2021-01-02 11:23:...|2021-01-02 00:00:00|\n", + "| 632739597| -290659081| NULL| www.eczachly.com| /sitemap.xml|2021-01-02 14:10:...|2021-01-02 00:00:00|\n", + "| 659201289| -290659081| NULL| www.eczachly.com|/blog/life-of-a-s...|2021-01-02 15:53:...|2021-01-02 00:00:00|\n", + "| 1072106763| -784483831|https://www.zachw...| www.zachwilson.tech| /contact|2021-01-02 09:37:...|2021-01-02 00:00:00|\n", + "| 1744817842| -223216734| NULL| www.zachwilson.tech|/graph/stock-tick...|2021-01-02 00:23:...|2021-01-02 00:00:00|\n", + "| 1445055201| 1957784035|https://www.zachw...| www.zachwilson.tech| /contact|2021-01-02 10:17:...|2021-01-02 00:00:00|\n", + "+-----------+-----------+--------------------+--------------------+--------------------+--------------------+-------------------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "# sortWithinPartitions - sorts within a partition, recommended \n", + "sorted = df.repartition(10, col(\"event_date\"))\\\n", + " .sortWithinPartitions(col(\"event_date\"), col(\"host\"))\\\n", + " .withColumn(\"event_time\", col(\"event_time\").cast(\"timestamp\")) \n", + "\n", + "# sort - does global sort across all data, not recommended\n", + "sortedTwo = df.repartition(10, col(\"event_date\"))\\\n", + " .sort(col(\"event_date\"), col(\"host\"))\\\n", + " .withColumn(\"event_time\", col(\"event_time\").cast(\"timestamp\")) \n", + "\n", + "sorted.show()\n", + "sortedTwo.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e75099ce-0e80-4903-8d3e-e4842a9a99f7", + "metadata": {}, + "outputs": [], + "source": [ + "# .sortWithinPartitions() sorts within partitions, whereas .sort() is a global sort, which is very slow\n", + "\n", + "# Note - exchange is synonymous with Shuffle\n", + "\n", + "# to undestand explain plan, read from bottom to top\n", + "# FileScan - read csv file\n", + "# Project - selects the column\n", + "# Exchange - means shuffles ~ repartition line\n", + "# Sort - sorts within partition or global - check true/false flag in explain plan\n", + "# Project - final select \n", + "\n", + "# true means GLOBAL sort enabled in second plan. Also, it will only work with one executor doing a global sort\n", + "# +- Sort [event_date#1835 ASC NULLS FIRST, host#1826 ASC NULLS FIRST], true, 0\n", + "# THIS `Exchange rangepartitioning` will be painful at scale - it will cause shuffle again after previous shuffle due to repartition\n", + "# +- Exchange rangepartitioning(event_date#1835 ASC NULLS FIRST, host#1826 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=2695]" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "5cc0bb63-2ef0-4a53-af25-881fbd785d55", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "== Physical Plan ==\n", + "AdaptiveSparkPlan isFinalPlan=false\n", + "+- Project [device_id#266, user_id#265, referrer#267, host#268, url#269, cast(event_time#270 as timestamp) AS event_time#1310, event_date#277, browser_family#322, os_family#323, device_type#306]\n", + " +- Sort [event_date#277 ASC NULLS FIRST, host#268 ASC NULLS FIRST], false, 0\n", + " +- Exchange hashpartitioning(event_date#277, 10), REPARTITION_BY_NUM, [plan_id=1661]\n", + " +- Project [device_id#266, user_id#265, referrer#267, host#268, url#269, event_time#270, event_date#277, browser_type#304 AS browser_family#322, os_type#305 AS os_family#323, device_type#306]\n", + " +- BroadcastHashJoin [device_id#266], [device_id#303], LeftOuter, BuildRight, false\n", + " :- Project [user_id#265, device_id#266, referrer#267, host#268, url#269, event_time#270, date_trunc(day, cast(event_time#270 as timestamp), Some(Etc/UTC)) AS event_date#277]\n", + " : +- FileScan csv [user_id#265,device_id#266,referrer#267,host#268,url#269,event_time#270] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/events.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n", + " +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=1658]\n", + " +- Filter isnotnull(device_id#303)\n", + " +- FileScan csv [device_id#303,browser_type#304,os_type#305,device_type#306] Batched: false, DataFilters: [isnotnull(device_id#303)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/devices.csv], PartitionFilters: [], PushedFilters: [IsNotNull(device_id)], ReadSchema: struct\n", + "\n", + "\n", + "== Physical Plan ==\n", + "AdaptiveSparkPlan isFinalPlan=false\n", + "+- Project [device_id#266, user_id#265, referrer#267, host#268, url#269, cast(event_time#270 as timestamp) AS event_time#1321, event_date#277, browser_family#322, os_family#323, device_type#306]\n", + " +- Sort [event_date#277 ASC NULLS FIRST, host#268 ASC NULLS FIRST], true, 0\n", + " +- Exchange rangepartitioning(event_date#277 ASC NULLS FIRST, host#268 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=1708]\n", + " +- Exchange hashpartitioning(event_date#277, 10), REPARTITION_BY_NUM, [plan_id=1706]\n", + " +- Project [device_id#266, user_id#265, referrer#267, host#268, url#269, event_time#270, event_date#277, browser_type#304 AS browser_family#322, os_type#305 AS os_family#323, device_type#306]\n", + " +- BroadcastHashJoin [device_id#266], [device_id#303], LeftOuter, BuildRight, false\n", + " :- Project [user_id#265, device_id#266, referrer#267, host#268, url#269, event_time#270, date_trunc(day, cast(event_time#270 as timestamp), Some(Etc/UTC)) AS event_date#277]\n", + " : +- FileScan csv [user_id#265,device_id#266,referrer#267,host#268,url#269,event_time#270] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/events.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n", + " +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=1703]\n", + " +- Filter isnotnull(device_id#303)\n", + " +- FileScan csv [device_id#303,browser_type#304,os_type#305,device_type#306] Batched: false, DataFilters: [isnotnull(device_id#303)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/home/iceberg/data/devices.csv], PartitionFilters: [], PushedFilters: [IsNotNull(device_id)], ReadSchema: struct\n", + "\n", + "\n" + ] + } + ], + "source": [ + "sorted = df.repartition(10, col(\"event_date\"))\\\n", + " .sortWithinPartitions(col(\"event_date\"), col(\"host\"))\\\n", + " .withColumn(\"event_time\", col(\"event_time\").cast(\"timestamp\")) \n", + "\n", + "sortedTwo = df.repartition(10, col(\"event_date\"))\\\n", + " .sort(col(\"event_date\"), col(\"host\"))\\\n", + " .withColumn(\"event_time\", col(\"event_time\").cast(\"timestamp\")) \n", + "\n", + "sorted.explain()\n", + "sortedTwo.explain()" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "1498a7f2-3a5c-4ee4-b936-e0d2b51f4eda", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------+-----------+--------+--------------------+----------+--------------------+-------------------+--------------+---------+-----------+\n", + "| device_id| user_id|referrer| host| url| event_time| event_date|browser_family|os_family|device_type|\n", + "+----------+-----------+--------+--------------------+----------+--------------------+-------------------+--------------+---------+-----------+\n", + "| 532630305| 1037710827| NULL| www.zachwilson.tech| /|2021-03-08 17:27:...|2021-03-08 00:00:00| Other| Other| Other|\n", + "| 532630305| 925588856| NULL| www.eczachly.com| /|2021-05-10 11:26:...|2021-05-10 00:00:00| Other| Other| Other|\n", + "| 532630305|-1180485268| NULL|admin.zachwilson....| /|2021-02-17 16:19:...|2021-02-17 00:00:00| Other| Other| Other|\n", + "| 532630305|-1044833855| NULL| www.zachwilson.tech| /|2021-09-24 15:53:...|2021-09-24 00:00:00| Other| Other| Other|\n", + "| 532630305| 747494706| NULL| www.zachwilson.tech| /|2021-09-26 16:03:...|2021-09-26 00:00:00| Other| Other| Other|\n", + "| 532630305| 747494706| NULL|admin.zachwilson....| /|2021-02-21 16:08:...|2021-02-21 00:00:00| Other| Other| Other|\n", + "| 532630305| -824540328| NULL|admin.zachwilson....| /|2021-09-28 17:23:...|2021-09-28 00:00:00| Other| Other| Other|\n", + "| 532630305| -824540328| NULL| www.eczachly.com| /|2021-09-29 01:22:...|2021-09-29 00:00:00| Other| Other| Other|\n", + "| 532630305| 1833036683| NULL|admin.zachwilson....| /|2021-01-24 03:15:...|2021-01-24 00:00:00| Other| Other| Other|\n", + "| 532630305|-2134824313| NULL| www.eczachly.com| /|2021-01-25 00:03:...|2021-01-25 00:00:00| Other| Other| Other|\n", + "|-906264142|-1809929467| NULL|admin.zachwilson....|/.git/HEAD|2021-02-22 01:36:...|2021-02-22 00:00:00| curl| Other| Other|\n", + "|-906264142| 2002285749| NULL| www.eczachly.com| /|2021-02-22 02:25:...|2021-02-22 00:00:00| curl| Other| Other|\n", + "| 532630305|-1562965412| NULL| www.zachwilson.tech| /|2021-01-30 20:46:...|2021-01-30 00:00:00| Other| Other| Other|\n", + "| 532630305|-1099860451| NULL| www.eczachly.com| /|2021-02-04 23:49:...|2021-02-04 00:00:00| Other| Other| Other|\n", + "|-906264142| 1246896869| NULL| www.zachwilson.tech| /|2021-02-22 02:50:...|2021-02-22 00:00:00| curl| Other| Other|\n", + "|-906264142| -629331502| NULL|admin.zachwilson....|/.git/HEAD|2021-02-22 23:51:...|2021-02-22 00:00:00| curl| Other| Other|\n", + "|-906264142|-1913422462| NULL| www.eczachly.com| /|2021-02-23 00:17:...|2021-02-23 00:00:00| curl| Other| Other|\n", + "| 532630305| 50429624| NULL| www.eczachly.com| /|2022-12-28 01:38:...|2022-12-28 00:00:00| Other| Other| Other|\n", + "| 532630305| 222389292| NULL| www.zachwilson.tech| /|2022-12-28 05:23:...|2022-12-28 00:00:00| Other| Other| Other|\n", + "| 532630305| -779924777| NULL| www.zachwilson.tech| /|2022-12-28 16:45:...|2022-12-28 00:00:00| Other| Other| Other|\n", + "+----------+-----------+--------+--------------------+----------+--------------------+-------------------+--------------+---------+-----------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import expr, col, lit\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "events = spark.read.option(\"header\", \"true\")\\\n", + " .csv(\"/home/iceberg/data/events.csv\")\\\n", + " .withColumn(\"event_date\", expr(\"DATE_TRUNC('day', event_time)\"))\n", + "\n", + "devices = spark.read.option(\"header\",\"true\")\\\n", + " .csv(\"/home/iceberg/data/devices.csv\")\n", + "\n", + "df = events.join(devices,on=\"device_id\",how=\"left\")\n", + "df = df.withColumnsRenamed({'browser_type': 'browser_family', 'os_type': 'os_family'})\n", + "\n", + "df.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "d800dca7-2737-4192-b5c0-c1806c105e15", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
" + ], + "text/plain": [ + "++\n", + "||\n", + "++\n", + "++" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "CREATE DATABASE IF NOT EXISTS bootcamp" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e83cd813-d5c0-4d67-8285-849b882b8bfa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
" + ], + "text/plain": [ + "++\n", + "||\n", + "++\n", + "++" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "DROP TABLE IF EXISTS bootcamp.events" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "b403139c-f342-426a-a6de-81cd86d1cd73", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
" + ], + "text/plain": [ + "++\n", + "||\n", + "++\n", + "++" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "DROP TABLE IF EXISTS bootcamp.events_sorted" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "d1b197a9-1b63-4130-acbe-01418eede0e5", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
" + ], + "text/plain": [ + "++\n", + "||\n", + "++\n", + "++" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "CREATE TABLE IF NOT EXISTS bootcamp.events (\n", + " url STRING,\n", + " referrer STRING,\n", + " browser_family STRING,\n", + " os_family STRING,\n", + " device_family STRING,\n", + " host STRING,\n", + " event_time TIMESTAMP,\n", + " event_date DATE\n", + ")\n", + "USING iceberg\n", + "PARTITIONED BY (event_date);\n" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "c40b143f-295e-4875-bd7f-12409312b800", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
" + ], + "text/plain": [ + "++\n", + "||\n", + "++\n", + "++" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "\n", + "CREATE TABLE IF NOT EXISTS bootcamp.events_sorted (\n", + " url STRING,\n", + " referrer STRING,\n", + " browser_family STRING,\n", + " os_family STRING,\n", + " device_family STRING,\n", + " host STRING,\n", + " event_time TIMESTAMP,\n", + " event_date DATE\n", + ")\n", + "USING iceberg\n", + "PARTITIONED BY (event_date);" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "00c86e79-a911-464c-ad58-acc92859dcc6", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
" + ], + "text/plain": [ + "++\n", + "||\n", + "++\n", + "++" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "\n", + "CREATE TABLE IF NOT EXISTS bootcamp.events_unsorted (\n", + " url STRING,\n", + " referrer STRING,\n", + " browser_family STRING,\n", + " os_family STRING,\n", + " device_family STRING,\n", + " host STRING,\n", + " event_time TIMESTAMP,\n", + " event_date DATE\n", + ")\n", + "USING iceberg\n", + "PARTITIONED BY (event_date);" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "2c1254bc-9ecf-4c86-bfd9-de81ecfbb78b", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "start_df = df.repartition(4, col(\"event_date\")).withColumn(\"event_time\", col(\"event_time\").cast(\"timestamp\")) \\\n", + " \n", + "first_sort_df = start_df.sortWithinPartitions(col(\"event_date\"), col(\"host\"))\n", + "\n", + "start_df.write.mode(\"overwrite\").saveAsTable(\"bootcamp.events_unsorted\")\n", + "first_sort_df.write.mode(\"overwrite\").saveAsTable(\"bootcamp.events_sorted\")" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "d207a11b-b0a4-4662-bbe9-747d8f67be7d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sizenum_filessorted
54449464sorted
55566724unsorted
" + ], + "text/plain": [ + "+---------+-----------+----------+\n", + "| size | num_files | sorted |\n", + "+---------+-----------+----------+\n", + "| 5444946 | 4 | sorted |\n", + "| 5556672 | 4 | unsorted |\n", + "+---------+-----------+----------+" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "SELECT SUM(file_size_in_bytes) as size, COUNT(1) as num_files, 'sorted' \n", + "FROM demo.bootcamp.events_sorted.files\n", + "\n", + "UNION ALL\n", + "SELECT SUM(file_size_in_bytes) as size, COUNT(1) as num_files, 'unsorted' \n", + "FROM demo.bootcamp.events_unsorted.files\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "a93db4d6-ac15-4d0e-83da-77b93ad618da", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sizenum_files
None0
" + ], + "text/plain": [ + "+------+-----------+\n", + "| size | num_files |\n", + "+------+-----------+\n", + "| None | 0 |\n", + "+------+-----------+" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "SELECT SUM(file_size_in_bytes) as size, COUNT(1) as num_files FROM demo.bootcamp.events.files;" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "7c3658c0", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "ename": "AnalysisException", + "evalue": "[TABLE_OR_VIEW_NOT_FOUND] The table or view `bootcamp`.`matches_bucketed`.`files` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.\nTo tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS.; line 1 pos 21;\n'Aggregate [unresolvedalias(count(1), None)]\n+- 'UnresolvedRelation [bootcamp, matches_bucketed, files], [], false\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAnalysisException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[45], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mget_ipython\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_cell_magic\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43msql\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mSELECT COUNT(1) FROM bootcamp.matches_bucketed.files\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2543\u001b[0m, in \u001b[0;36mInteractiveShell.run_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m 2541\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuiltin_trap:\n\u001b[1;32m 2542\u001b[0m args \u001b[38;5;241m=\u001b[39m (magic_arg_s, cell)\n\u001b[0;32m-> 2543\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2545\u001b[0m \u001b[38;5;66;03m# The code below prevents the output from being displayed\u001b[39;00m\n\u001b[1;32m 2546\u001b[0m \u001b[38;5;66;03m# when using magics with decorator @output_can_be_silenced\u001b[39;00m\n\u001b[1;32m 2547\u001b[0m \u001b[38;5;66;03m# when the last Python token in the expression is a ';'.\u001b[39;00m\n\u001b[1;32m 2548\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(fn, magic\u001b[38;5;241m.\u001b[39mMAGIC_OUTPUT_CAN_BE_SILENCED, \u001b[38;5;28;01mFalse\u001b[39;00m):\n", + "File \u001b[0;32m~/.ipython/profile_default/startup/00-prettytables.py:81\u001b[0m, in \u001b[0;36msql\u001b[0;34m(line, cell)\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _to_table(df, num_rows\u001b[38;5;241m=\u001b[39margs\u001b[38;5;241m.\u001b[39mlimit)\n\u001b[1;32m 80\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _to_table(\u001b[43mspark\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msql\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcell\u001b[49m\u001b[43m)\u001b[49m)\n", + "File \u001b[0;32m/opt/spark/python/pyspark/sql/session.py:1631\u001b[0m, in \u001b[0;36mSparkSession.sql\u001b[0;34m(self, sqlQuery, args, **kwargs)\u001b[0m\n\u001b[1;32m 1627\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jvm \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 1628\u001b[0m litArgs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jvm\u001b[38;5;241m.\u001b[39mPythonUtils\u001b[38;5;241m.\u001b[39mtoArray(\n\u001b[1;32m 1629\u001b[0m [_to_java_column(lit(v)) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m (args \u001b[38;5;129;01mor\u001b[39;00m [])]\n\u001b[1;32m 1630\u001b[0m )\n\u001b[0;32m-> 1631\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m DataFrame(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jsparkSession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msql\u001b[49m\u001b[43m(\u001b[49m\u001b[43msqlQuery\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlitArgs\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m 1632\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 1633\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(kwargs) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n", + "File \u001b[0;32m/opt/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/java_gateway.py:1322\u001b[0m, in \u001b[0;36mJavaMember.__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1316\u001b[0m command \u001b[38;5;241m=\u001b[39m proto\u001b[38;5;241m.\u001b[39mCALL_COMMAND_NAME \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1317\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_header \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1318\u001b[0m args_command \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1319\u001b[0m proto\u001b[38;5;241m.\u001b[39mEND_COMMAND_PART\n\u001b[1;32m 1321\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgateway_client\u001b[38;5;241m.\u001b[39msend_command(command)\n\u001b[0;32m-> 1322\u001b[0m return_value \u001b[38;5;241m=\u001b[39m \u001b[43mget_return_value\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1323\u001b[0m \u001b[43m \u001b[49m\u001b[43manswer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtarget_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1325\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m temp_arg \u001b[38;5;129;01min\u001b[39;00m temp_args:\n\u001b[1;32m 1326\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(temp_arg, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_detach\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n", + "File \u001b[0;32m/opt/spark/python/pyspark/errors/exceptions/captured.py:185\u001b[0m, in \u001b[0;36mcapture_sql_exception..deco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m 181\u001b[0m converted \u001b[38;5;241m=\u001b[39m convert_exception(e\u001b[38;5;241m.\u001b[39mjava_exception)\n\u001b[1;32m 182\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(converted, UnknownException):\n\u001b[1;32m 183\u001b[0m \u001b[38;5;66;03m# Hide where the exception came from that shows a non-Pythonic\u001b[39;00m\n\u001b[1;32m 184\u001b[0m \u001b[38;5;66;03m# JVM exception message.\u001b[39;00m\n\u001b[0;32m--> 185\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m converted \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 187\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m\n", + "\u001b[0;31mAnalysisException\u001b[0m: [TABLE_OR_VIEW_NOT_FOUND] The table or view `bootcamp`.`matches_bucketed`.`files` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.\nTo tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS.; line 1 pos 21;\n'Aggregate [unresolvedalias(count(1), None)]\n+- 'UnresolvedRelation [bootcamp, matches_bucketed, files], [], false\n" + ] + } + ], + "source": [ + "%%sql \n", + "SELECT COUNT(1) FROM bootcamp.matches_bucketed.files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "674f173d-86ca-4637-9332-37ce04a81451", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "591ba0b7-2e6f-420a-960d-95e5734b0166", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
contentfile_pathfile_formatspec_idpartitionrecord_countfile_size_in_bytescolumn_sizesvalue_countsnull_value_countsnan_value_countslower_boundsupper_boundskey_metadatasplit_offsetsequality_idssort_order_idreferenced_data_filecontent_offsetcontent_size_in_bytesreadable_metrics
0s3://warehouse/bootcamp/events_sorted/data/00000-159-8e8be00f-b845-451f-b45a-031e97f06779-0-00001.parquetPARQUET1Row(event_date=None)893911032261{1: 107466, 2: 61022, 3: 11455, 4: 12926, 6: 7383, 7: 426449, 8: 2293, 9: 77425, 10: 310063, 11: 10711}{1: 89391, 2: 89391, 3: 89391, 4: 89391, 6: 89391, 7: 89391, 8: 89391, 9: 89391, 10: 89391, 11: 89391}{1: 0, 2: 46359, 3: 0, 4: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 1, 11: 0}{}{1: bytearray(b'/'), 2: bytearray(b'52.20.78.240'), 3: bytearray(b'%E3%82%A6%E3%82%'), 4: bytearray(b'Android'), 6: bytearray(b'aashish.techcrea'), 7: bytearray(b' \\xba\\xe7\\xb8\\xa8\\xb8\\x05\\x00'), 8: bytearray(b'\\x00\\xa0&\\xb4\\xa8\\xb8\\x05\\x00'), 9: bytearray(b'-100210680'), 10: bytearray(b'-1000095488'), 11: bytearray(b'17MB150WB')}{1: bytearray(b'/zzageqnf.php?Fp'), 2: bytearray(b'zachwilson.tech'), 3: bytearray(b'webprosbot'), 4: bytearray(b'iOS'), 6: bytearray(b'zachwilson.techd'), 7: bytearray(b'\\xe8\\xb0\\x1b\\x8ec\\x03\\x06\\x00'), 8: bytearray(b'\\x00\\xe0dqO\\x03\\x06\\x00'), 9: bytearray(b'999535123'), 10: bytearray(b'999884938'), 11: bytearray(b'vivo $2')}None[4]None0NoneNoneNoneRow(browser_family=Row(column_size=11455, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='%E3%82%A6%E3%82%', upper_bound='webprosbot'), device_id=Row(column_size=77425, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='-100210680', upper_bound='999535123'), device_type=Row(column_size=10711, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='17MB150WB', upper_bound='vivo $2'), event_date=Row(column_size=2293, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 12, 0, 0), upper_bound=datetime.datetime(2023, 8, 20, 0, 0)), event_time=Row(column_size=426449, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 12, 0, 1, 19, 764000), upper_bound=datetime.datetime(2023, 8, 20, 23, 59, 41, 89000)), host=Row(column_size=7383, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='aashish.techcrea', upper_bound='zachwilson.techd'), os_family=Row(column_size=12926, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='Android', upper_bound='iOS'), referrer=Row(column_size=61022, value_count=89391, null_value_count=46359, nan_value_count=None, lower_bound='52.20.78.240', upper_bound='zachwilson.tech'), url=Row(column_size=107466, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='/', upper_bound='/zzageqnf.php?Fp'), user_id=Row(column_size=310063, value_count=89391, null_value_count=1, nan_value_count=None, lower_bound='-1000095488', upper_bound='999884938'))
0s3://warehouse/bootcamp/events_sorted/data/00001-160-8e8be00f-b845-451f-b45a-031e97f06779-0-00001.parquetPARQUET1Row(event_date=None)992321165546{1: 142178, 2: 67363, 3: 11914, 4: 16543, 6: 9119, 7: 475862, 8: 2373, 9: 86514, 10: 337013, 11: 11522}{1: 99232, 2: 99232, 3: 99232, 4: 99232, 6: 99232, 7: 99232, 8: 99232, 9: 99232, 10: 99232, 11: 99232}{1: 0, 2: 49299, 3: 0, 4: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 58, 11: 0}{}{1: bytearray(b'"/?""<?=print(93'), 2: bytearray(b'"https://www.goo'), 3: bytearray(b') Bot'), 4: bytearray(b'Android'), 6: bytearray(b'abhishekanand.te'), 7: bytearray(b'(\\x83\\xb2EX\\xb8\\x05\\x00'), 8: bytearray(b'\\x00 \\xc9<X\\xb8\\x05\\x00'), 9: bytearray(b'-100210680'), 10: bytearray(b'-1000370060'), 11: bytearray(b'13 Pro Max')}{1: bytearray(b'/zz.php'), 2: bytearray(b'zachwilson.tech'), 3: bytearray(b'webprosbot'), 4: bytearray(b'iOS'), 6: bytearray(b'zsavi524.techcrf'), 7: bytearray(b'\\x88\\xb8\\x07P;\\x03\\x06\\x00'), 8: bytearray(b"\\x00 \\xb65\\'\\x03\\x06\\x00"), 9: bytearray(b'999535123'), 10: bytearray(b'999956796'), 11: bytearray(b'vivo $2')}None[4]None0NoneNoneNoneRow(browser_family=Row(column_size=11914, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound=') Bot', upper_bound='webprosbot'), device_id=Row(column_size=86514, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound='-100210680', upper_bound='999535123'), device_type=Row(column_size=11522, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound='13 Pro Max', upper_bound='vivo $2'), event_date=Row(column_size=2373, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 8, 0, 0), upper_bound=datetime.datetime(2023, 8, 18, 0, 0)), event_time=Row(column_size=475862, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 8, 0, 2, 29, 513000), upper_bound=datetime.datetime(2023, 8, 18, 23, 59, 0, 901000)), host=Row(column_size=9119, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound='abhishekanand.te', upper_bound='zsavi524.techcrf'), os_family=Row(column_size=16543, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound='Android', upper_bound='iOS'), referrer=Row(column_size=67363, value_count=99232, null_value_count=49299, nan_value_count=None, lower_bound='"https://www.goo', upper_bound='zachwilson.tech'), url=Row(column_size=142178, value_count=99232, null_value_count=0, nan_value_count=None, lower_bound='"/?""<?=print(93', upper_bound='/zz.php'), user_id=Row(column_size=337013, value_count=99232, null_value_count=58, nan_value_count=None, lower_bound='-1000370060', upper_bound='999956796'))
0s3://warehouse/bootcamp/events_sorted/data/00002-161-8e8be00f-b845-451f-b45a-031e97f06779-0-00001.parquetPARQUET1Row(event_date=None)939561354864{1: 345976, 2: 86611, 3: 10966, 4: 12894, 6: 8590, 7: 447243, 8: 2037, 9: 86885, 10: 336434, 11: 12138}{1: 93956, 2: 93956, 3: 93956, 4: 93956, 6: 93956, 7: 93956, 8: 93956, 9: 93956, 10: 93956, 11: 93956}{1: 0, 2: 48227, 3: 1, 4: 1, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0, 11: 1}{}{1: bytearray(b'"/?""<?=print(93'), 2: bytearray(b'"https://www.goo'), 3: bytearray(b') Bot'), 4: bytearray(b'Android'), 6: bytearray(b'ablumhardt.techc'), 7: bytearray(b'\\x18\\xe8_\\xb2\\xf3\\xb7\\x05\\x00'), 8: bytearray(b'\\x00@\\x94\\xa7\\xf3\\xb7\\x05\\x00'), 9: bytearray(b'-1000866068'), 10: bytearray(b'-1000675882'), 11: bytearray(b'ALP-AL00')}{1: bytearray(b'/zz/address.php@'), 2: bytearray(b'zachwilson.tech'), 3: bytearray(b'webprosbot'), 4: bytearray(b'webOS'), 6: bytearray(b'zzz.techcreator/'), 7: bytearray(b'HE\\xdbM\\xb3\\x03\\x06\\x00'), 8: bytearray(b'\\x00`\\xc2\\xe8\\x9f\\x03\\x06\\x00'), 9: bytearray(b'998961543'), 10: bytearray(b'999956796'), 11: bytearray(b'vivo $2')}None[4]None0NoneNoneNoneRow(browser_family=Row(column_size=10966, value_count=93956, null_value_count=1, nan_value_count=None, lower_bound=') Bot', upper_bound='webprosbot'), device_id=Row(column_size=86885, value_count=93956, null_value_count=0, nan_value_count=None, lower_bound='-1000866068', upper_bound='998961543'), device_type=Row(column_size=12138, value_count=93956, null_value_count=1, nan_value_count=None, lower_bound='ALP-AL00', upper_bound='vivo $2'), event_date=Row(column_size=2037, value_count=93956, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 3, 0, 0), upper_bound=datetime.datetime(2023, 8, 24, 0, 0)), event_time=Row(column_size=447243, value_count=93956, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 3, 0, 3, 1, 119000), upper_bound=datetime.datetime(2023, 8, 24, 23, 8, 20, 509000)), host=Row(column_size=8590, value_count=93956, null_value_count=0, nan_value_count=None, lower_bound='ablumhardt.techc', upper_bound='zzz.techcreator/'), os_family=Row(column_size=12894, value_count=93956, null_value_count=1, nan_value_count=None, lower_bound='Android', upper_bound='webOS'), referrer=Row(column_size=86611, value_count=93956, null_value_count=48227, nan_value_count=None, lower_bound='"https://www.goo', upper_bound='zachwilson.tech'), url=Row(column_size=345976, value_count=93956, null_value_count=0, nan_value_count=None, lower_bound='"/?""<?=print(93', upper_bound='/zz/address.php@'), user_id=Row(column_size=336434, value_count=93956, null_value_count=0, nan_value_count=None, lower_bound='-1000675882', upper_bound='999956796'))
0s3://warehouse/bootcamp/events_sorted/data/00003-162-8e8be00f-b845-451f-b45a-031e97f06779-0-00001.parquetPARQUET1Row(event_date=None)1222351542439{1: 284361, 2: 87456, 3: 11525, 4: 16894, 6: 9348, 7: 558677, 8: 2178, 9: 110135, 10: 442488, 11: 13563}{1: 122235, 2: 122235, 3: 122235, 4: 122235, 6: 122235, 7: 122235, 8: 122235, 9: 122235, 10: 122235, 11: 122235}{1: 0, 2: 53009, 3: 0, 4: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 8, 11: 0}{}{1: bytearray(b'/'), 2: bytearray(b'3.220.57.224'), 3: bytearray(b') Bot'), 4: bytearray(b'Android'), 6: bytearray(b'accc.techcreator'), 7: bytearray(b'@n.\\xbd\\xdf\\xb7\\x05\\x00'), 8: bytearray(b'\\x00\\xe0\\xbc\\x89\\xdf\\xb7\\x05\\x00'), 9: bytearray(b'-1001669954'), 10: bytearray(b'-1000015881'), 11: bytearray(b'$2')}{1: bytearray(b'/zz.php'), 2: bytearray(b'zachwilson.tech'), 3: bytearray(b'webprosbot'), 4: bytearray(b'iOS'), 6: bytearray(b'zachwilson.techd'), 7: bytearray(b'\\xd8\\xaf\\x9a\\xe8\\x9f\\x03\\x06\\x00'), 8: bytearray(b'\\x00\\x00\\xeb\\xca\\x8b\\x03\\x06\\x00'), 9: bytearray(b'998766634'), 10: bytearray(b'999882344'), 11: bytearray(b'vivo $2')}None[4]None0NoneNoneNoneRow(browser_family=Row(column_size=11525, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound=') Bot', upper_bound='webprosbot'), device_id=Row(column_size=110135, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound='-1001669954', upper_bound='998766634'), device_type=Row(column_size=13563, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound='$2', upper_bound='vivo $2'), event_date=Row(column_size=2178, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 2, 0, 0), upper_bound=datetime.datetime(2023, 8, 23, 0, 0)), event_time=Row(column_size=558677, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 2, 0, 14, 23, 80000), upper_bound=datetime.datetime(2023, 8, 23, 23, 59, 57, 399000)), host=Row(column_size=9348, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound='accc.techcreator', upper_bound='zachwilson.techd'), os_family=Row(column_size=16894, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound='Android', upper_bound='iOS'), referrer=Row(column_size=87456, value_count=122235, null_value_count=53009, nan_value_count=None, lower_bound='3.220.57.224', upper_bound='zachwilson.tech'), url=Row(column_size=284361, value_count=122235, null_value_count=0, nan_value_count=None, lower_bound='/', upper_bound='/zz.php'), user_id=Row(column_size=442488, value_count=122235, null_value_count=8, nan_value_count=None, lower_bound='-1000015881', upper_bound='999882344'))
" + ], + "text/plain": [ + "+---------+-----------------------------------------------------------------------------------------------------------+-------------+---------+----------------------+--------------+--------------------+----------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------+------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+---------------+--------------+---------------+----------------------+----------------+-----------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| content | file_path | file_format | spec_id | partition | record_count | file_size_in_bytes | column_sizes | value_counts | null_value_counts | nan_value_counts | lower_bounds | upper_bounds | key_metadata | split_offsets | equality_ids | sort_order_id | referenced_data_file | content_offset | content_size_in_bytes | readable_metrics |\n", + "+---------+-----------------------------------------------------------------------------------------------------------+-------------+---------+----------------------+--------------+--------------------+----------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------+------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+---------------+--------------+---------------+----------------------+----------------+-----------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| 0 | s3://warehouse/bootcamp/events_sorted/data/00000-159-8e8be00f-b845-451f-b45a-031e97f06779-0-00001.parquet | PARQUET | 1 | Row(event_date=None) | 89391 | 1032261 | {1: 107466, 2: 61022, 3: 11455, 4: 12926, 6: 7383, 7: 426449, 8: 2293, 9: 77425, 10: 310063, 11: 10711} | {1: 89391, 2: 89391, 3: 89391, 4: 89391, 6: 89391, 7: 89391, 8: 89391, 9: 89391, 10: 89391, 11: 89391} | {1: 0, 2: 46359, 3: 0, 4: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 1, 11: 0} | {} | {1: bytearray(b'/'), 2: bytearray(b'52.20.78.240'), 3: bytearray(b'%E3%82%A6%E3%82%'), 4: bytearray(b'Android'), 6: bytearray(b'aashish.techcrea'), 7: bytearray(b' \\xba\\xe7\\xb8\\xa8\\xb8\\x05\\x00'), 8: bytearray(b'\\x00\\xa0&\\xb4\\xa8\\xb8\\x05\\x00'), 9: bytearray(b'-100210680'), 10: bytearray(b'-1000095488'), 11: bytearray(b'17MB150WB')} | {1: bytearray(b'/zzageqnf.php?Fp'), 2: bytearray(b'zachwilson.tech'), 3: bytearray(b'webprosbot'), 4: bytearray(b'iOS'), 6: bytearray(b'zachwilson.techd'), 7: bytearray(b'\\xe8\\xb0\\x1b\\x8ec\\x03\\x06\\x00'), 8: bytearray(b'\\x00\\xe0dqO\\x03\\x06\\x00'), 9: bytearray(b'999535123'), 10: bytearray(b'999884938'), 11: bytearray(b'vivo $2')} | None | [4] | None | 0 | None | None | None | Row(browser_family=Row(column_size=11455, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='%E3%82%A6%E3%82%', upper_bound='webprosbot'), device_id=Row(column_size=77425, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='-100210680', upper_bound='999535123'), device_type=Row(column_size=10711, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='17MB150WB', upper_bound='vivo $2'), event_date=Row(column_size=2293, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 12, 0, 0), upper_bound=datetime.datetime(2023, 8, 20, 0, 0)), event_time=Row(column_size=426449, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound=datetime.datetime(2021, 1, 12, 0, 1, 19, 764000), upper_bound=datetime.datetime(2023, 8, 20, 23, 59, 41, 89000)), host=Row(column_size=7383, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='aashish.techcrea', upper_bound='zachwilson.techd'), os_family=Row(column_size=12926, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='Android', upper_bound='iOS'), referrer=Row(column_size=61022, value_count=89391, null_value_count=46359, nan_value_count=None, lower_bound='52.20.78.240', upper_bound='zachwilson.tech'), url=Row(column_size=107466, value_count=89391, null_value_count=0, nan_value_count=None, lower_bound='/', upper_bound='/zzageqnf.php?Fp'), user_id=Row(column_size=310063, value_count=89391, null_value_count=1, nan_value_count=None, lower_bound='-1000095488', upper_bound='999884938')) |\n", + "| 0 | s3://warehouse/bootcamp/events_sorted/data/00001-160-8e8be00f-b845-451f-b45a-031e97f06779-0-00001.parquet | PARQUET | 1 | Row(event_date=None) | 99232 | 1165546 | {1: 142178, 2: 67363, 3: 11914, 4: 16543, 6: 9119, 7: 475862, 8: 2373, 9: 86514, 10: 337013, 11: 11522} | {1: 99232, 2: 99232, 3: 99232, 4: 99232, 6: 99232, 7: 99232, 8: 99232, 9: 99232, 10: 99232, 11: 99232} | {1: 0, 2: 49299, 3: 0, 4: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 58, 11: 0} | {} | {1: bytearray(b'\"/?\"\"\n", + " \n", + " \n", + " device_id\n", + " user_id\n", + " referrer\n", + " host\n", + " url\n", + " event_time\n", + " event_date\n", + " browser_family\n", + " os_family\n", + " device_type\n", + " \n", + " \n", + " \n", + " \n", + " 532630305\n", + " -488618451\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-12 10:22:17.016000\n", + " 2021-01-12 00:00:00\n", + " Other\n", + " Other\n", + " Other\n", + " \n", + " \n", + " 589185851\n", + " -21136712\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-12 18:49:28.425000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Linux\n", + " Other\n", + " \n", + " \n", + " 589185851\n", + " -414920062\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-12 18:54:30.995000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Linux\n", + " Other\n", + " \n", + " \n", + " 589185851\n", + " -414920062\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-12 19:56:56.809000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Linux\n", + " Other\n", + " \n", + " \n", + " 589185851\n", + " -694958230\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-12 20:08:15.964000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Linux\n", + " Other\n", + " \n", + " \n", + " -290659081\n", + " 2105351485\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 04:44:23.791000\n", + " 2021-01-12 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 01:13:53.762000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.eczachly.com/\n", + " www.eczachly.com\n", + " /blog\n", + " 2021-01-12 01:13:56.914000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.eczachly.com/blog\n", + " www.eczachly.com\n", + " /graphs\n", + " 2021-01-12 01:13:58.899000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.eczachly.com/graphs\n", + " www.eczachly.com\n", + " /graphs\n", + " 2021-01-12 01:14:01.017000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.eczachly.com/graphs\n", + " www.eczachly.com\n", + " /about\n", + " 2021-01-12 01:14:02.615000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.eczachly.com/about\n", + " www.eczachly.com\n", + " /contact\n", + " 2021-01-12 01:14:40.193000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " 1885594540\n", + " -505490192\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 04:04:30.165000\n", + " 2021-01-12 00:00:00\n", + " MJ12bot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -290659081\n", + " -2142732711\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 05:14:02.344000\n", + " 2021-01-12 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -290659081\n", + " 2075040354\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 07:51:23.359000\n", + " 2021-01-12 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " 1088283544\n", + " 210988258\n", + " None\n", + " www.eczachly.com\n", + " /blog/life-of-a-silicon-valley-big-data-engineer-3-critical-soft-skills-for-success\n", + " 2021-01-12 10:55:59.598000\n", + " 2021-01-12 00:00:00\n", + " PetalBot\n", + " Android\n", + " Generic Smartphone\n", + " \n", + " \n", + " -290659081\n", + " 2075040354\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 11:21:43.159000\n", + " 2021-01-12 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -290659081\n", + " -2142732711\n", + " None\n", + " www.eczachly.com\n", + " /sitemap.xml\n", + " 2021-01-12 14:11:11.874000\n", + " 2021-01-12 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " 1088283544\n", + " -2086279634\n", + " None\n", + " www.eczachly.com\n", + " /graphs\n", + " 2021-01-12 15:22:44.781000\n", + " 2021-01-12 00:00:00\n", + " PetalBot\n", + " Android\n", + " Generic Smartphone\n", + " \n", + " \n", + " -158310583\n", + " 1228096790\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 17:31:41.575000\n", + " 2021-01-12 00:00:00\n", + " PetalBot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " 532630305\n", + " -1188109378\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 19:45:31.342000\n", + " 2021-01-12 00:00:00\n", + " Other\n", + " Other\n", + " Other\n", + " \n", + " \n", + " 1858056360\n", + " -1414351633\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 23:07:54.189000\n", + " 2021-01-12 00:00:00\n", + " YandexBot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " 1858056360\n", + " -1414351633\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-12 23:07:58.003000\n", + " 2021-01-12 00:00:00\n", + " YandexBot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -2012543895\n", + " -1017906782\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 17:35:40.684000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -2012543895\n", + " -1132914083\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 00:01:19.764000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -1395826244\n", + " 824962582\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 15:16:49.303000\n", + " 2021-01-12 00:00:00\n", + " SemrushBot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " 1088283544\n", + " 633710304\n", + " None\n", + " www.zachwilson.tech\n", + " /blog/so-what-exactly-is-a-data-engineer\n", + " 2021-01-12 00:48:33.860000\n", + " 2021-01-12 00:00:00\n", + " PetalBot\n", + " Android\n", + " Generic Smartphone\n", + " \n", + " \n", + " -1883465659\n", + " 1363688232\n", + " None\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 01:01:10.078000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 01:06:16.150000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:08:24.956000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:08:25.468000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 01:08:33.199000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -1883465659\n", + " 1091280702\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 05:33:52.143000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:08:36.318000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " 1645401075\n", + " 966119024\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 13:27:45.903000\n", + " 2021-01-12 00:00:00\n", + " Firefox\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:08:36.500000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 01:08:38.831000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:08:43.130000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:08:43.311000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " None\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:10:33.604000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 01:10:37.578000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 01:10:40.612000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 01:11:45.400000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/contact\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:11:48.023000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:11:49.373000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog/test-blog-post\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 01:12:31.449000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/contact\n", + " www.zachwilson.tech\n", + " /graphs\n", + " 2021-01-12 01:12:34.417000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog/test-blog-post\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 01:13:14.940000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:18.187000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:19.263000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:19.792000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:19.982000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:20.172000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:20.510000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:20.341000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /blog/test-blog-post\n", + " 2021-01-12 01:13:20.712000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/contact\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 01:13:23.527000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 01:26:09.367000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 01:26:11.524000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " https://www.zachwilson.tech/blog\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 01:32:37.695000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 01:33:34.627000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 01:34:19.227000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " -843023486\n", + " -2116612468\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 01:34:47.807000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Mac OS X\n", + " Other\n", + " \n", + " \n", + " 543928648\n", + " -777218977\n", + " https://www.zachwilson.tech/wp-login.php\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 01:39:08.884000\n", + " 2021-01-12 00:00:00\n", + " Firefox\n", + " Ubuntu\n", + " Other\n", + " \n", + " \n", + " -2012543895\n", + " 365926159\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 03:42:42.238000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -1883465659\n", + " 1930062144\n", + " None\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 04:53:18.755000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -2012543895\n", + " -1017906782\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 07:24:49.838000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -1883465659\n", + " 1558866943\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 07:35:01.242000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -1883465659\n", + " -1631735911\n", + " None\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 07:45:03.198000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " 532630305\n", + " 747494706\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 07:54:31.829000\n", + " 2021-01-12 00:00:00\n", + " Other\n", + " Other\n", + " Other\n", + " \n", + " \n", + " -158310583\n", + " 1152852815\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 08:13:52.596000\n", + " 2021-01-12 00:00:00\n", + " PetalBot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -2012543895\n", + " 842220057\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 08:14:02.125000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -1883465659\n", + " 842220057\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 08:14:02.157000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -1395826244\n", + " 1771129212\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 15:16:48.802000\n", + " 2021-01-12 00:00:00\n", + " SemrushBot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -1648112917\n", + " 1251343619\n", + " http://zachwilson.tech\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 10:23:17.485000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -1648112917\n", + " 1251343619\n", + " https://www.zachwilson.tech/\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 10:23:18.100000\n", + " 2021-01-12 00:00:00\n", + " Chrome\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -1259816303\n", + " -1769609334\n", + " None\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 11:17:32.148000\n", + " 2021-01-12 00:00:00\n", + " QQ Browser\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " 1645401075\n", + " 966119024\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 13:27:44.870000\n", + " 2021-01-12 00:00:00\n", + " Firefox\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -2012543895\n", + " 842220057\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 13:34:40.547000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -1395826244\n", + " -274134875\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 15:16:50.033000\n", + " 2021-01-12 00:00:00\n", + " SemrushBot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " 1081314922\n", + " -1685318372\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 15:45:16.745000\n", + " 2021-01-12 00:00:00\n", + " Mobile Safari\n", + " iOS\n", + " iPhone\n", + " \n", + " \n", + " 1081314922\n", + " 1231174081\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 16:06:30.519000\n", + " 2021-01-12 00:00:00\n", + " Mobile Safari\n", + " iOS\n", + " iPhone\n", + " \n", + " \n", + " -1883465659\n", + " 718479942\n", + " None\n", + " www.zachwilson.tech\n", + " /blog\n", + " 2021-01-12 17:25:51.782000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -2012543895\n", + " -1858543565\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 17:35:40.747000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " 851565341\n", + " 1454953440\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 19:51:13.020000\n", + " 2021-01-12 00:00:00\n", + " Mobile Safari\n", + " iOS\n", + " iPhone\n", + " \n", + " \n", + " -1883465659\n", + " -1741507611\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 20:36:53.427000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " 2066336516\n", + " 883939942\n", + " http://uswm.semxiu.com/blog/102439346/7-proven-ways-to-increase-website-traffic/\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 20:38:37.012000\n", + " 2021-01-12 00:00:00\n", + " Firefox\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -2012543895\n", + " 835375946\n", + " None\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 20:56:50.324000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -1883465659\n", + " 1805707385\n", + " None\n", + " www.zachwilson.tech\n", + " /contact\n", + " 2021-01-12 21:57:19.765000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -1883465659\n", + " 1091280702\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 22:07:21.675000\n", + " 2021-01-12 00:00:00\n", + " Googlebot\n", + " Android\n", + " Spider\n", + " \n", + " \n", + " -290659081\n", + " -309207910\n", + " None\n", + " www.zachwilson.tech\n", + " /\n", + " 2021-01-12 22:35:42.231000\n", + " 2021-01-12 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -290659081\n", + " -1730861305\n", + " None\n", + " www.zachwilson.tech\n", + " /about\n", + " 2021-01-12 22:35:54.115000\n", + " 2021-01-12 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -643696601\n", + " 1272828233\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-14 10:40:31.324000\n", + " 2021-01-14 00:00:00\n", + " Chrome\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -643696601\n", + " 1272828233\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-14 12:02:34.478000\n", + " 2021-01-14 00:00:00\n", + " Chrome\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -643696601\n", + " 1272828233\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-14 17:56:50.003000\n", + " 2021-01-14 00:00:00\n", + " Chrome\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -643696601\n", + " 1272828233\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-14 19:42:51.705000\n", + " 2021-01-14 00:00:00\n", + " Chrome\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -643696601\n", + " 1272828233\n", + " None\n", + " admin.zachwilson.tech\n", + " /\n", + " 2021-01-14 21:57:39.320000\n", + " 2021-01-14 00:00:00\n", + " Chrome\n", + " Windows\n", + " Other\n", + " \n", + " \n", + " -290659081\n", + " -1820003202\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-14 22:20:39.780000\n", + " 2021-01-14 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -290659081\n", + " 2075040354\n", + " None\n", + " www.eczachly.com\n", + " /\n", + " 2021-01-14 02:44:38.950000\n", + " 2021-01-14 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + " -290659081\n", + " -2142732711\n", + " None\n", + " www.eczachly.com\n", + " /sitemap.xml\n", + " 2021-01-14 05:30:58.067000\n", + " 2021-01-14 00:00:00\n", + " bingbot\n", + " Other\n", + " Spider\n", + " \n", + " \n", + "" + ], + "text/plain": [ + "+-------------+-------------+----------------------------------------------------------------------------------+-----------------------+-------------------------------------------------------------------------------------+----------------------------+---------------------+----------------+-----------+--------------------+\n", + "| device_id | user_id | referrer | host | url | event_time | event_date | browser_family | os_family | device_type |\n", + "+-------------+-------------+----------------------------------------------------------------------------------+-----------------------+-------------------------------------------------------------------------------------+----------------------------+---------------------+----------------+-----------+--------------------+\n", + "| 532630305 | -488618451 | None | admin.zachwilson.tech | / | 2021-01-12 10:22:17.016000 | 2021-01-12 00:00:00 | Other | Other | Other |\n", + "| 589185851 | -21136712 | None | admin.zachwilson.tech | / | 2021-01-12 18:49:28.425000 | 2021-01-12 00:00:00 | Chrome | Linux | Other |\n", + "| 589185851 | -414920062 | None | admin.zachwilson.tech | / | 2021-01-12 18:54:30.995000 | 2021-01-12 00:00:00 | Chrome | Linux | Other |\n", + "| 589185851 | -414920062 | None | admin.zachwilson.tech | / | 2021-01-12 19:56:56.809000 | 2021-01-12 00:00:00 | Chrome | Linux | Other |\n", + "| 589185851 | -694958230 | None | admin.zachwilson.tech | / | 2021-01-12 20:08:15.964000 | 2021-01-12 00:00:00 | Chrome | Linux | Other |\n", + "| -290659081 | 2105351485 | None | www.eczachly.com | / | 2021-01-12 04:44:23.791000 | 2021-01-12 00:00:00 | bingbot | Other | Spider |\n", + "| -843023486 | -2116612468 | None | www.eczachly.com | / | 2021-01-12 01:13:53.762000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.eczachly.com/ | www.eczachly.com | /blog | 2021-01-12 01:13:56.914000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.eczachly.com/blog | www.eczachly.com | /graphs | 2021-01-12 01:13:58.899000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.eczachly.com/graphs | www.eczachly.com | /graphs | 2021-01-12 01:14:01.017000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.eczachly.com/graphs | www.eczachly.com | /about | 2021-01-12 01:14:02.615000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.eczachly.com/about | www.eczachly.com | /contact | 2021-01-12 01:14:40.193000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| 1885594540 | -505490192 | None | www.eczachly.com | / | 2021-01-12 04:04:30.165000 | 2021-01-12 00:00:00 | MJ12bot | Other | Spider |\n", + "| -290659081 | -2142732711 | None | www.eczachly.com | / | 2021-01-12 05:14:02.344000 | 2021-01-12 00:00:00 | bingbot | Other | Spider |\n", + "| -290659081 | 2075040354 | None | www.eczachly.com | / | 2021-01-12 07:51:23.359000 | 2021-01-12 00:00:00 | bingbot | Other | Spider |\n", + "| 1088283544 | 210988258 | None | www.eczachly.com | /blog/life-of-a-silicon-valley-big-data-engineer-3-critical-soft-skills-for-success | 2021-01-12 10:55:59.598000 | 2021-01-12 00:00:00 | PetalBot | Android | Generic Smartphone |\n", + "| -290659081 | 2075040354 | None | www.eczachly.com | / | 2021-01-12 11:21:43.159000 | 2021-01-12 00:00:00 | bingbot | Other | Spider |\n", + "| -290659081 | -2142732711 | None | www.eczachly.com | /sitemap.xml | 2021-01-12 14:11:11.874000 | 2021-01-12 00:00:00 | bingbot | Other | Spider |\n", + "| 1088283544 | -2086279634 | None | www.eczachly.com | /graphs | 2021-01-12 15:22:44.781000 | 2021-01-12 00:00:00 | PetalBot | Android | Generic Smartphone |\n", + "| -158310583 | 1228096790 | None | www.eczachly.com | / | 2021-01-12 17:31:41.575000 | 2021-01-12 00:00:00 | PetalBot | Other | Spider |\n", + "| 532630305 | -1188109378 | None | www.eczachly.com | / | 2021-01-12 19:45:31.342000 | 2021-01-12 00:00:00 | Other | Other | Other |\n", + "| 1858056360 | -1414351633 | None | www.eczachly.com | / | 2021-01-12 23:07:54.189000 | 2021-01-12 00:00:00 | YandexBot | Other | Spider |\n", + "| 1858056360 | -1414351633 | None | www.eczachly.com | / | 2021-01-12 23:07:58.003000 | 2021-01-12 00:00:00 | YandexBot | Other | Spider |\n", + "| -2012543895 | -1017906782 | None | www.zachwilson.tech | / | 2021-01-12 17:35:40.684000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| -2012543895 | -1132914083 | None | www.zachwilson.tech | /about | 2021-01-12 00:01:19.764000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| -1395826244 | 824962582 | None | www.zachwilson.tech | / | 2021-01-12 15:16:49.303000 | 2021-01-12 00:00:00 | SemrushBot | Other | Spider |\n", + "| 1088283544 | 633710304 | None | www.zachwilson.tech | /blog/so-what-exactly-is-a-data-engineer | 2021-01-12 00:48:33.860000 | 2021-01-12 00:00:00 | PetalBot | Android | Generic Smartphone |\n", + "| -1883465659 | 1363688232 | None | www.zachwilson.tech | /contact | 2021-01-12 01:01:10.078000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -843023486 | -2116612468 | None | www.zachwilson.tech | / | 2021-01-12 01:06:16.150000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/ | www.zachwilson.tech | /blog | 2021-01-12 01:08:24.956000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog | 2021-01-12 01:08:25.468000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | / | 2021-01-12 01:08:33.199000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -1883465659 | 1091280702 | None | www.zachwilson.tech | / | 2021-01-12 05:33:52.143000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/ | www.zachwilson.tech | /blog | 2021-01-12 01:08:36.318000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| 1645401075 | 966119024 | None | www.zachwilson.tech | /about | 2021-01-12 13:27:45.903000 | 2021-01-12 00:00:00 | Firefox | Windows | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog | 2021-01-12 01:08:36.500000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | / | 2021-01-12 01:08:38.831000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/ | www.zachwilson.tech | /blog | 2021-01-12 01:08:43.130000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog | 2021-01-12 01:08:43.311000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | None | www.zachwilson.tech | /blog | 2021-01-12 01:10:33.604000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | / | 2021-01-12 01:10:37.578000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | / | 2021-01-12 01:10:40.612000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/ | www.zachwilson.tech | /contact | 2021-01-12 01:11:45.400000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/contact | www.zachwilson.tech | /blog | 2021-01-12 01:11:48.023000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:11:49.373000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog/test-blog-post | www.zachwilson.tech | /contact | 2021-01-12 01:12:31.449000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/contact | www.zachwilson.tech | /graphs | 2021-01-12 01:12:34.417000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog/test-blog-post | www.zachwilson.tech | /contact | 2021-01-12 01:13:14.940000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:18.187000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:19.263000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:19.792000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:19.982000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:20.172000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:20.510000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:20.341000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /blog/test-blog-post | 2021-01-12 01:13:20.712000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/contact | www.zachwilson.tech | /contact | 2021-01-12 01:13:23.527000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | None | www.zachwilson.tech | / | 2021-01-12 01:26:09.367000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/ | www.zachwilson.tech | /blog | 2021-01-12 01:26:11.524000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | https://www.zachwilson.tech/blog | www.zachwilson.tech | /about | 2021-01-12 01:32:37.695000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | None | www.zachwilson.tech | /about | 2021-01-12 01:33:34.627000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | None | www.zachwilson.tech | /about | 2021-01-12 01:34:19.227000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| -843023486 | -2116612468 | None | www.zachwilson.tech | /about | 2021-01-12 01:34:47.807000 | 2021-01-12 00:00:00 | Chrome | Mac OS X | Other |\n", + "| 543928648 | -777218977 | https://www.zachwilson.tech/wp-login.php | www.zachwilson.tech | / | 2021-01-12 01:39:08.884000 | 2021-01-12 00:00:00 | Firefox | Ubuntu | Other |\n", + "| -2012543895 | 365926159 | None | www.zachwilson.tech | / | 2021-01-12 03:42:42.238000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| -1883465659 | 1930062144 | None | www.zachwilson.tech | /blog | 2021-01-12 04:53:18.755000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -2012543895 | -1017906782 | None | www.zachwilson.tech | / | 2021-01-12 07:24:49.838000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| -1883465659 | 1558866943 | None | www.zachwilson.tech | / | 2021-01-12 07:35:01.242000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -1883465659 | -1631735911 | None | www.zachwilson.tech | /contact | 2021-01-12 07:45:03.198000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| 532630305 | 747494706 | None | www.zachwilson.tech | / | 2021-01-12 07:54:31.829000 | 2021-01-12 00:00:00 | Other | Other | Other |\n", + "| -158310583 | 1152852815 | None | www.zachwilson.tech | / | 2021-01-12 08:13:52.596000 | 2021-01-12 00:00:00 | PetalBot | Other | Spider |\n", + "| -2012543895 | 842220057 | None | www.zachwilson.tech | / | 2021-01-12 08:14:02.125000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| -1883465659 | 842220057 | None | www.zachwilson.tech | /about | 2021-01-12 08:14:02.157000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -1395826244 | 1771129212 | None | www.zachwilson.tech | / | 2021-01-12 15:16:48.802000 | 2021-01-12 00:00:00 | SemrushBot | Other | Spider |\n", + "| -1648112917 | 1251343619 | http://zachwilson.tech | www.zachwilson.tech | / | 2021-01-12 10:23:17.485000 | 2021-01-12 00:00:00 | Chrome | Windows | Other |\n", + "| -1648112917 | 1251343619 | https://www.zachwilson.tech/ | www.zachwilson.tech | /about | 2021-01-12 10:23:18.100000 | 2021-01-12 00:00:00 | Chrome | Windows | Other |\n", + "| -1259816303 | -1769609334 | None | www.zachwilson.tech | /contact | 2021-01-12 11:17:32.148000 | 2021-01-12 00:00:00 | QQ Browser | Windows | Other |\n", + "| 1645401075 | 966119024 | None | www.zachwilson.tech | / | 2021-01-12 13:27:44.870000 | 2021-01-12 00:00:00 | Firefox | Windows | Other |\n", + "| -2012543895 | 842220057 | None | www.zachwilson.tech | / | 2021-01-12 13:34:40.547000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| -1395826244 | -274134875 | None | www.zachwilson.tech | / | 2021-01-12 15:16:50.033000 | 2021-01-12 00:00:00 | SemrushBot | Other | Spider |\n", + "| 1081314922 | -1685318372 | None | www.zachwilson.tech | / | 2021-01-12 15:45:16.745000 | 2021-01-12 00:00:00 | Mobile Safari | iOS | iPhone |\n", + "| 1081314922 | 1231174081 | None | www.zachwilson.tech | / | 2021-01-12 16:06:30.519000 | 2021-01-12 00:00:00 | Mobile Safari | iOS | iPhone |\n", + "| -1883465659 | 718479942 | None | www.zachwilson.tech | /blog | 2021-01-12 17:25:51.782000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -2012543895 | -1858543565 | None | www.zachwilson.tech | / | 2021-01-12 17:35:40.747000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| 851565341 | 1454953440 | None | www.zachwilson.tech | / | 2021-01-12 19:51:13.020000 | 2021-01-12 00:00:00 | Mobile Safari | iOS | iPhone |\n", + "| -1883465659 | -1741507611 | None | www.zachwilson.tech | /about | 2021-01-12 20:36:53.427000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| 2066336516 | 883939942 | http://uswm.semxiu.com/blog/102439346/7-proven-ways-to-increase-website-traffic/ | www.zachwilson.tech | / | 2021-01-12 20:38:37.012000 | 2021-01-12 00:00:00 | Firefox | Windows | Other |\n", + "| -2012543895 | 835375946 | None | www.zachwilson.tech | /contact | 2021-01-12 20:56:50.324000 | 2021-01-12 00:00:00 | Googlebot | Other | Spider |\n", + "| -1883465659 | 1805707385 | None | www.zachwilson.tech | /contact | 2021-01-12 21:57:19.765000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -1883465659 | 1091280702 | None | www.zachwilson.tech | / | 2021-01-12 22:07:21.675000 | 2021-01-12 00:00:00 | Googlebot | Android | Spider |\n", + "| -290659081 | -309207910 | None | www.zachwilson.tech | / | 2021-01-12 22:35:42.231000 | 2021-01-12 00:00:00 | bingbot | Other | Spider |\n", + "| -290659081 | -1730861305 | None | www.zachwilson.tech | /about | 2021-01-12 22:35:54.115000 | 2021-01-12 00:00:00 | bingbot | Other | Spider |\n", + "| -643696601 | 1272828233 | None | admin.zachwilson.tech | / | 2021-01-14 10:40:31.324000 | 2021-01-14 00:00:00 | Chrome | Windows | Other |\n", + "| -643696601 | 1272828233 | None | admin.zachwilson.tech | / | 2021-01-14 12:02:34.478000 | 2021-01-14 00:00:00 | Chrome | Windows | Other |\n", + "| -643696601 | 1272828233 | None | admin.zachwilson.tech | / | 2021-01-14 17:56:50.003000 | 2021-01-14 00:00:00 | Chrome | Windows | Other |\n", + "| -643696601 | 1272828233 | None | admin.zachwilson.tech | / | 2021-01-14 19:42:51.705000 | 2021-01-14 00:00:00 | Chrome | Windows | Other |\n", + "| -643696601 | 1272828233 | None | admin.zachwilson.tech | / | 2021-01-14 21:57:39.320000 | 2021-01-14 00:00:00 | Chrome | Windows | Other |\n", + "| -290659081 | -1820003202 | None | www.eczachly.com | / | 2021-01-14 22:20:39.780000 | 2021-01-14 00:00:00 | bingbot | Other | Spider |\n", + "| -290659081 | 2075040354 | None | www.eczachly.com | / | 2021-01-14 02:44:38.950000 | 2021-01-14 00:00:00 | bingbot | Other | Spider |\n", + "| -290659081 | -2142732711 | None | www.eczachly.com | /sitemap.xml | 2021-01-14 05:30:58.067000 | 2021-01-14 00:00:00 | bingbot | Other | Spider |\n", + "+-------------+-------------+----------------------------------------------------------------------------------+-----------------------+-------------------------------------------------------------------------------------+----------------------------+---------------------+----------------+-----------+--------------------+" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM demo.bootcamp.events_sorted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a083daf-d141-47fc-9661-db22c2313a88", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}