Yuriy Gavrilov

Welcome to my personal place for love, peace and happiness 🤖

Follow this blog
Later Ctrl + ↑
 No comments   2023   Cloud dbeaver
 No comments   2023   Mitino
 No comments   2023   Sochi S7
 No comments   2023   Bike
 No comments   2023  
 No comments   2023   NFT

Чтение avro в spark из s3

import pyspark
import os

S3_ACCESS_KEY = os.environ.get(“S3_ACCESS_KEY”)
S3_BUCKET = os.environ.get(“S3_BUCKET”)
S3_SECRET_KEY = os.environ.get(“S3_SECRET_KEY”)
S3_ENDPOINT = os.environ.get(“S3_ENDPOINT”)

This cell may take some time to run the first time, as it must download the necessary spark jars

conf = pyspark.SparkConf()

IF YOU ARE USING THE SPARK CONTAINERS, UNCOMMENT THE LINE BELOW TO OFFLOAD EXECUTION OF SPARK TASKS TO SPARK CONTAINERS

#conf.setMaster(“spark://spark:7077”)

conf.set(“spark.jars.packages”, ‘org.apache.hadoop:hadoop-aws:3.3.1,io.delta:delta-core_2.12:2.1.0,org.apache.spark:spark-avro_2.12:3.3.2’)

conf.set(‘spark.hadoop.fs.s3a.aws.credentials.provider’, ‘org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider’)

conf.set(‘spark.hadoop.fs.s3a.endpoint’, S3_ENDPOINT)
conf.set(‘spark.hadoop.fs.s3a.access.key’, S3_ACCESS_KEY)
conf.set(‘spark.hadoop.fs.s3a.secret.key’, S3_SECRET_KEY)
conf.set(‘spark.hadoop.fs.s3a.path.style.access’, “true”)
conf.set(“spark.sql.extensions”, “io.delta.sql.DeltaSparkSessionExtension”)
conf.set(“spark.sql.catalog.spark_catalog”, “org.apache.spark.sql.delta.catalog.DeltaCatalog”)

sc = pyspark.SparkContext(conf=conf)

sc.setLogLevel(“INFO”)

spark = pyspark.sql.SparkSession(sc)

df = spark.read.format(“avro”).load(f“s3a://{S3_BUCKET}/person2.avro”)

 No comments   2023   spark

Загрузка данных iceberg в Trino

Создание схем

CREATE SCHEMA delta.my_schema_d
WITH (location = ‘s3a://test/delta_p’);

CREATE SCHEMA hive.my_schema_h
WITH (location = ‘s3a://test/hive_p’);

CREATE SCHEMA iceberg.my_schema_i
WITH (location = ‘s3a://test/iceberg_p’);

Создание таблицы на базе CSV

create table hive.my_schema_h.taxi_p (
VendorID bigint,
tpep_pickup_datetime varchar,
tpep_dropoff_datetime varchar,
passenger_count bigint,
trip_distance decimal(20,2),
RatecodeID bigint,
store_and_fwd_flag varchar,
PULocationID bigint,
DOLocationID bigint,
payment_type bigint,
fare_amount decimal(20,2),
extra decimal(20,2),
mta_tax decimal(20,2),
tip_amount decimal(20,2),
tolls_amount decimal(20,2),
improvement_surcharge decimal(20,2),
total_amount decimal(20,2))
WITH (
format = ‘TEXTFILE’,
textfile_field_separator = ‘,’,
skip_header_line_count=1
);

Создание таблицы iceberg

create table iceberg.my_schema_i.taxi_i (
VendorID bigint,
tpep_pickup_datetime varchar,
tpep_dropoff_datetime varchar,
passenger_count bigint,
trip_distance decimal(20,2),
RatecodeID bigint,
store_and_fwd_flag varchar,
PULocationID bigint,
DOLocationID bigint,
payment_type bigint,
fare_amount decimal(20,2),
extra decimal(20,2),
mta_tax decimal(20,2),
tip_amount decimal(20,2),
tolls_amount decimal(20,2),
improvement_surcharge decimal(20,2),
total_amount decimal(20,2))

Запись таблицы iceberg

insert into iceberg.my_schema_i.taxi_i select * from hive.my_schema_h.taxi_p

 No comments   2023   SQL

Классификация в Trino

CREATE SCHEMA delta.my_schema_d
WITH (location = ‘s3a://test/delta_p’);

CREATE TABLE delta.my_schema_d.iris (
id int
, sepal_length double
, sepal_width double
, petal_length double
, petal_width double
, species varchar
);

INSERT INTO delta.my_schema_d.iris VALUES
(1, 5.1, 3.5, 1.4, 0.2, ‘Iris-setosa’),
(2, 4.9, 3, 1.4, 0.2, ‘Iris-setosa’),
(3, 4.7, 3.2, 1.3, 0.2, ‘Iris-setosa’),
(4, 4.6, 3.1, 1.5, 0.2, ‘Iris-setosa’),
(5, 5, 3.6, 1.4, 0.2, ‘Iris-setosa’),
(6, 5.4, 3.9, 1.7, 0.4, ‘Iris-setosa’),
(7, 4.6, 3.4, 1.4, 0.3, ‘Iris-setosa’),
(8, 5, 3.4, 1.5, 0.2, ‘Iris-setosa’),
(9, 4.4, 2.9, 1.4, 0.2, ‘Iris-setosa’),
(10, 4.9, 3.1, 1.5, 0.1, ‘Iris-setosa’),
(11, 5.4, 3.7, 1.5, 0.2, ‘Iris-setosa’),
(12, 4.8, 3.4, 1.6, 0.2, ‘Iris-setosa’),
(13, 4.8, 3, 1.4, 0.1, ‘Iris-setosa’),
(14, 4.3, 3, 1.1, 0.1, ‘Iris-setosa’),
(15, 5.8, 4, 1.2, 0.2, ‘Iris-setosa’),
(16, 5.7, 4.4, 1.5, 0.4, ‘Iris-setosa’),
(17, 5.4, 3.9, 1.3, 0.4, ‘Iris-setosa’),
(18, 5.1, 3.5, 1.4, 0.3, ‘Iris-setosa’),
(19, 5.7, 3.8, 1.7, 0.3, ‘Iris-setosa’),
(20, 5.1, 3.8, 1.5, 0.3, ‘Iris-setosa’),
(21, 5.4, 3.4, 1.7, 0.2, ‘Iris-setosa’),
(22, 5.1, 3.7, 1.5, 0.4, ‘Iris-setosa’),
(23, 4.6, 3.6, 1, 0.2, ‘Iris-setosa’),
(24, 5.1, 3.3, 1.7, 0.5, ‘Iris-setosa’),
(25, 4.8, 3.4, 1.9, 0.2, ‘Iris-setosa’),
(26, 5, 3, 1.6, 0.2, ‘Iris-setosa’),
(27, 5, 3.4, 1.6, 0.4, ‘Iris-setosa’),
(28, 5.2, 3.5, 1.5, 0.2, ‘Iris-setosa’),
(29, 5.2, 3.4, 1.4, 0.2, ‘Iris-setosa’),
(30, 4.7, 3.2, 1.6, 0.2, ‘Iris-setosa’),
(31, 4.8, 3.1, 1.6, 0.2, ‘Iris-setosa’),
(32, 5.4, 3.4, 1.5, 0.4, ‘Iris-setosa’),
(33, 5.2, 4.1, 1.5, 0.1, ‘Iris-setosa’),
(34, 5.5, 4.2, 1.4, 0.2, ‘Iris-setosa’),
(35, 4.9, 3.1, 1.5, 0.1, ‘Iris-setosa’),
(36, 5, 3.2, 1.2, 0.2, ‘Iris-setosa’),
(37, 5.5, 3.5, 1.3, 0.2, ‘Iris-setosa’),
(38, 4.9, 3.1, 1.5, 0.1, ‘Iris-setosa’),
(39, 4.4, 3, 1.3, 0.2, ‘Iris-setosa’),
(40, 5.1, 3.4, 1.5, 0.2, ‘Iris-setosa’),
(41, 5, 3.5, 1.3, 0.3, ‘Iris-setosa’),
(42, 4.5, 2.3, 1.3, 0.3, ‘Iris-setosa’),
(43, 4.4, 3.2, 1.3, 0.2, ‘Iris-setosa’),
(44, 5, 3.5, 1.6, 0.6, ‘Iris-setosa’),
(45, 5.1, 3.8, 1.9, 0.4, ‘Iris-setosa’),
(46, 4.8, 3, 1.4, 0.3, ‘Iris-setosa’),
(47, 5.1, 3.8, 1.6, 0.2, ‘Iris-setosa’),
(48, 4.6, 3.2, 1.4, 0.2, ‘Iris-setosa’),
(49, 5.3, 3.7, 1.5, 0.2, ‘Iris-setosa’),
(50, 5, 3.3, 1.4, 0.2, ‘Iris-setosa’),
(51, 7, 3.2, 4.7, 1.4, ‘Iris-versicolor’),
(52, 6.4, 3.2, 4.5, 1.5, ‘Iris-versicolor’),
(53, 6.9, 3.1, 4.9, 1.5, ‘Iris-versicolor’),
(54, 5.5, 2.3, 4, 1.3, ‘Iris-versicolor’),
(55, 6.5, 2.8, 4.6, 1.5, ‘Iris-versicolor’),
(56, 5.7, 2.8, 4.5, 1.3, ‘Iris-versicolor’),
(57, 6.3, 3.3, 4.7, 1.6, ‘Iris-versicolor’),
(58, 4.9, 2.4, 3.3, 1, ‘Iris-versicolor’),
(59, 6.6, 2.9, 4.6, 1.3, ‘Iris-versicolor’),
(60, 5.2, 2.7, 3.9, 1.4, ‘Iris-versicolor’),
(61, 5, 2, 3.5, 1, ‘Iris-versicolor’),
(62, 5.9, 3, 4.2, 1.5, ‘Iris-versicolor’),
(63, 6, 2.2, 4, 1, ‘Iris-versicolor’),
(64, 6.1, 2.9, 4.7, 1.4, ‘Iris-versicolor’),
(65, 5.6, 2.9, 3.6, 1.3, ‘Iris-versicolor’),
(66, 6.7, 3.1, 4.4, 1.4, ‘Iris-versicolor’),
(67, 5.6, 3, 4.5, 1.5, ‘Iris-versicolor’),
(68, 5.8, 2.7, 4.1, 1, ‘Iris-versicolor’),
(69, 6.2, 2.2, 4.5, 1.5, ‘Iris-versicolor’),
(70, 5.6, 2.5, 3.9, 1.1, ‘Iris-versicolor’),
(71, 5.9, 3.2, 4.8, 1.8, ‘Iris-versicolor’),
(72, 6.1, 2.8, 4, 1.3, ‘Iris-versicolor’),
(73, 6.3, 2.5, 4.9, 1.5, ‘Iris-versicolor’),
(74, 6.1, 2.8, 4.7, 1.2, ‘Iris-versicolor’),
(75, 6.4, 2.9, 4.3, 1.3, ‘Iris-versicolor’),
(76, 6.6, 3, 4.4, 1.4, ‘Iris-versicolor’),
(77, 6.8, 2.8, 4.8, 1.4, ‘Iris-versicolor’),
(78, 6.7, 3, 5, 1.7, ‘Iris-versicolor’),
(79, 6, 2.9, 4.5, 1.5, ‘Iris-versicolor’),
(80, 5.7, 2.6, 3.5, 1, ‘Iris-versicolor’),
(81, 5.5, 2.4, 3.8, 1.1, ‘Iris-versicolor’),
(82, 5.5, 2.4, 3.7, 1, ‘Iris-versicolor’),
(83, 5.8, 2.7, 3.9, 1.2, ‘Iris-versicolor’),
(84, 6, 2.7, 5.1, 1.6, ‘Iris-versicolor’),
(85, 5.4, 3, 4.5, 1.5, ‘Iris-versicolor’),
(86, 6, 3.4, 4.5, 1.6, ‘Iris-versicolor’),
(87, 6.7, 3.1, 4.7, 1.5, ‘Iris-versicolor’),
(88, 6.3, 2.3, 4.4, 1.3, ‘Iris-versicolor’),
(89, 5.6, 3, 4.1, 1.3, ‘Iris-versicolor’),
(90, 5.5, 2.5, 4, 1.3, ‘Iris-versicolor’),
(91, 5.5, 2.6, 4.4, 1.2, ‘Iris-versicolor’),
(92, 6.1, 3, 4.6, 1.4, ‘Iris-versicolor’),
(93, 5.8, 2.6, 4, 1.2, ‘Iris-versicolor’),
(94, 5, 2.3, 3.3, 1, ‘Iris-versicolor’),
(95, 5.6, 2.7, 4.2, 1.3, ‘Iris-versicolor’),
(96, 5.7, 3, 4.2, 1.2, ‘Iris-versicolor’),
(97, 5.7, 2.9, 4.2, 1.3, ‘Iris-versicolor’),
(98, 6.2, 2.9, 4.3, 1.3, ‘Iris-versicolor’),
(99, 5.1, 2.5, 3, 1.1, ‘Iris-versicolor’),
(100, 5.7, 2.8, 4.1, 1.3, ‘Iris-versicolor’),
(101, 6.3, 3.3, 6, 2.5, ‘Iris-virginica’),
(102, 5.8, 2.7, 5.1, 1.9, ‘Iris-virginica’),
(103, 7.1, 3, 5.9, 2.1, ‘Iris-virginica’),
(104, 6.3, 2.9, 5.6, 1.8, ‘Iris-virginica’),
(105, 6.5, 3, 5.8, 2.2, ‘Iris-virginica’),
(106, 7.6, 3, 6.6, 2.1, ‘Iris-virginica’),
(107, 4.9, 2.5, 4.5, 1.7, ‘Iris-virginica’),
(108, 7.3, 2.9, 6.3, 1.8, ‘Iris-virginica’),
(109, 6.7, 2.5, 5.8, 1.8, ‘Iris-virginica’),
(110, 7.2, 3.6, 6.1, 2.5, ‘Iris-virginica’),
(111, 6.5, 3.2, 5.1, 2, ‘Iris-virginica’),
(112, 6.4, 2.7, 5.3, 1.9, ‘Iris-virginica’),
(113, 6.8, 3, 5.5, 2.1, ‘Iris-virginica’),
(114, 5.7, 2.5, 5, 2, ‘Iris-virginica’),
(115, 5.8, 2.8, 5.1, 2.4, ‘Iris-virginica’),
(116, 6.4, 3.2, 5.3, 2.3, ‘Iris-virginica’),
(117, 6.5, 3, 5.5, 1.8, ‘Iris-virginica’),
(118, 7.7, 3.8, 6.7, 2.2, ‘Iris-virginica’),
(119, 7.7, 2.6, 6.9, 2.3, ‘Iris-virginica’),
(120, 6, 2.2, 5, 1.5, ‘Iris-virginica’),
(121, 6.9, 3.2, 5.7, 2.3, ‘Iris-virginica’),
(122, 5.6, 2.8, 4.9, 2, ‘Iris-virginica’),
(123, 7.7, 2.8, 6.7, 2, ‘Iris-virginica’),
(124, 6.3, 2.7, 4.9, 1.8, ‘Iris-virginica’),
(125, 6.7, 3.3, 5.7, 2.1, ‘Iris-virginica’),
(126, 7.2, 3.2, 6, 1.8, ‘Iris-virginica’),
(127, 6.2, 2.8, 4.8, 1.8, ‘Iris-virginica’),
(128, 6.1, 3, 4.9, 1.8, ‘Iris-virginica’),
(129, 6.4, 2.8, 5.6, 2.1, ‘Iris-virginica’),
(130, 7.2, 3, 5.8, 1.6, ‘Iris-virginica’),
(131, 7.4, 2.8, 6.1, 1.9, ‘Iris-virginica’),
(132, 7.9, 3.8, 6.4, 2, ‘Iris-virginica’),
(133, 6.4, 2.8, 5.6, 2.2, ‘Iris-virginica’),
(134, 6.3, 2.8, 5.1, 1.5, ‘Iris-virginica’),
(135, 6.1, 2.6, 5.6, 1.4, ‘Iris-virginica’),
(136, 7.7, 3, 6.1, 2.3, ‘Iris-virginica’),
(137, 6.3, 3.4, 5.6, 2.4, ‘Iris-virginica’),
(138, 6.4, 3.1, 5.5, 1.8, ‘Iris-virginica’),
(139, 6, 3, 4.8, 1.8, ‘Iris-virginica’),
(140, 6.9, 3.1, 5.4, 2.1, ‘Iris-virginica’),
(141, 6.7, 3.1, 5.6, 2.4, ‘Iris-virginica’),
(142, 6.9, 3.1, 5.1, 2.3, ‘Iris-virginica’),
(143, 5.8, 2.7, 5.1, 1.9, ‘Iris-virginica’),
(144, 6.8, 3.2, 5.9, 2.3, ‘Iris-virginica’),
(145, 6.7, 3.3, 5.7, 2.5, ‘Iris-virginica’),
(146, 6.7, 3, 5.2, 2.3, ‘Iris-virginica’),
(147, 6.3, 2.5, 5, 1.9, ‘Iris-virginica’),
(148, 6.5, 3, 5.2, 2, ‘Iris-virginica’),
(149, 6.2, 3.4, 5.4, 2.3, ‘Iris-virginica’),
(150, 5.9, 3, 5.1, 1.8, ‘Iris-virginica’);

select classify(features(5.8, 3, 5.1, 1.8), model) AS prediction
FROM (
SELECT learn_classifier(species, features(sepal_length, sepal_width, petal_length, petal_width)) AS model FROM delta.my_schema_d.iris
) t;

Iris-virginica

 No comments   2023   SQL

Чтение json в spark из s3

import pyspark
import os

S3_ACCESS_KEY = os.environ.get(“S3_ACCESS_KEY”)
S3_BUCKET = os.environ.get(“S3_BUCKET”)
S3_SECRET_KEY = os.environ.get(“S3_SECRET_KEY”)
S3_ENDPOINT = os.environ.get(“S3_ENDPOINT”)

This cell may take some time to run the first time, as it must download the necessary spark jars

conf = pyspark.SparkConf()

IF YOU ARE USING THE SPARK CONTAINERS, UNCOMMENT THE LINE BELOW TO OFFLOAD EXECUTION OF SPARK TASKS TO SPARK CONTAINERS

#conf.setMaster(“spark://spark:7077”)

conf.set(“spark.jars.packages”, ‘org.apache.hadoop:hadoop-aws:3.3.1,io.delta:delta-core_2.12:2.1.0,org.apache.spark:spark-avro_2.12:3.3.2’)

conf.set(‘spark.hadoop.fs.s3a.aws.credentials.provider’, ‘org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider’)

conf.set(‘spark.hadoop.fs.s3a.endpoint’, S3_ENDPOINT)
conf.set(‘spark.hadoop.fs.s3a.access.key’, S3_ACCESS_KEY)
conf.set(‘spark.hadoop.fs.s3a.secret.key’, S3_SECRET_KEY)
conf.set(‘spark.hadoop.fs.s3a.path.style.access’, “true”)
conf.set(“spark.sql.extensions”, “io.delta.sql.DeltaSparkSessionExtension”)
conf.set(“spark.sql.catalog.spark_catalog”, “org.apache.spark.sql.delta.catalog.DeltaCatalog”)

sc = pyspark.SparkContext(conf=conf)

sc.setLogLevel(“INFO”)

spark = pyspark.sql.SparkSession(sc)

df = spark.read.format(‘org.apache.spark.sql.json’).load(f“s3a://{S3_BUCKET}/apple3.json”)
df.show()

 No comments   2023   spark
Earlier Ctrl + ↓
© Yuriy Gavrilov - B[u]g - for charity.gavrilov.eth, 2022–2025 RSS
Powered by Aegea