Техника запроса

SEL ECT * FR OM STRING_SPLIT('Lorem ipsum dolor sit amet.', ' ', 1);

SEL ECT ProductId, Name, value  
FR OM Product  
    CROSS APPLY STRING_SPLIT(Tags, ',');

SEL ECT <КолонкаСФирмой>, <ИмяКолонкиФИОстрокой>, value  
FR OM tab  
    CROSS APPLY STRING_SPLIT(<ИмяКолонкиФИОстрокой>, ' ')

WITH vt_data as
(SEL ECT t.f as f, t.str||' ' as str 
FR OM
	(SEL ECT 'Фирма 1' as f, 'Иван Петров Ольга Иванова' as str
	UNI ON ALL SEL ECT 'Фирма 2', 'Петр Петров'
	UNI ON ALL SELECT 'Фирма 3', 'Олег Иванов Петр Петров Анна Петрова'
	) as t
)

, vt as
(SELECT
	 t.f
	,substr(t.str, instr(t.str, ' ')+1) as str
	,substr(t.str, 0, instr(t.str, ' ')) as slovo
	,1 as npp
FR OM
	vt_data as t
	
uni on ALL

SEL ECT
	 t.f
	,substr(t.str, instr(t.str, ' ')+1)
	,substr(t.str, 0, instr(t.str, ' '))
	,t.npp + 1 as npp
FR OM vt as t
WH ERE instr(t.str, ' ')<>0
)

SEL ECT t.f as f, t.FI as FI
FR OM
	(SEL ECT
		 t.f as f
		,t.slovo || ' ' || LEAD(t.slovo) over(PARTITION by t.f ORDER by t.npp) as FI
		,t.npp as npp
	FR OM
		vt as t
	) as t
WH ERE t.npp%2<>0
ORDER by t.f, t.npp

Результат
f	                FI
Фирма 1	Иван Петров
Фирма 1	Ольга Иванова
Фирма 2	Петр Петров
Фирма 3	Олег Иванов
Фирма 3	Петр Петров
Фирма 3	Анна Петрова

# Выбор одного столбца
data.sel ect('sector').show(5)

# Выбор нескольких столбцов
data.select(['open', 'close', 'adjusted']).show(5)

# Использование CASE WHEN в PySpark
sql = """
CASE
    WHEN (lang LIKE 'c') OR (lang LIKE 'cpp') THEN 'C/C++'
    WHEN (lang LIKE 'py') THEN 'Python'
    ELSE 'unknown'
END"""
df.withColumn('FullLang', F.expr(sql)).show()

# Использование JOIN в PySpark
df.join(df2, df.name == df2.name, 'outer').select(df.name, df2.height).sort(desc("name")).show()

# Loop Through if you dont know recusrsive depth
while True:
  this_level = 'level_{}'.format(i)
  next_level = 'level_{}'.format(i+1)
  emp_level = employees \
    .withColumnRenamed('EMPLOYEE_NUMBER', this_level) \
    .withColumnRenamed('MANAGER_EMPLOYEE_NUMBER', next_level)
  empDF = empDF.join(emp_level, on=this_level, how='left')
  
  # Check if DF is empty. Break loop if empty, Otherwise continue with next level
  if empDF.where(col(next_level).isNotNull()).count() == 0:
    break
  else:
    i += 1

# Sort columns and show
empDF.sort('level_0').select('level_0').show()

spark.sql("select lang fr om prog_info").show()

sc.sql("SEL ECT * FR OM SPLIT('f f f f f',' ')").show()