1use std::fs::File;
2use std::path::Path;
3use std::sync::Arc;
4
5use arrow::array::StringArray;
6use arrow::array::{Array, BooleanArray, PrimitiveArray, RecordBatch, UInt64Array};
7use arrow::compute;
8use arrow::datatypes::{DataType, Field, Schema, SchemaRef, UInt64Type};
9use itertools::Itertools;
10use log::info;
11use parquet::arrow::ArrowWriter;
12use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
13use parquet::file::properties::WriterProperties;
14
15use crate::common::graph::Graph;
16use crate::common::node_indexing::UmiToNodeIndexMapping;
17use crate::common::node_partitioning::NodePartitioning;
18use crate::common::types::{EdgeWeight, NodeIdx, UMI, UMIPair};
19
20pub struct ParquetUMIPairIter {
23 reader: ParquetRecordBatchReader,
24
25 expected_size: i64,
26
27 col_src: Option<PrimitiveArray<UInt64Type>>,
30 col_dst: Option<PrimitiveArray<UInt64Type>>,
31
32 current_idx: usize,
34 batch_len: usize,
35}
36
37impl ParquetUMIPairIter {
38 pub fn new<P: AsRef<Path>>(path: P) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
39 let file = File::open(path)?;
40
41 let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
42 let num_rows = builder.metadata().file_metadata().num_rows();
43 let reader = builder.with_batch_size(8192).build()?;
44
45 Ok(Self {
46 reader,
47 expected_size: num_rows,
48 col_src: None,
49 col_dst: None,
50 current_idx: 0,
51 batch_len: 0,
52 })
53 }
54
55 fn load_next_batch(&mut self) -> bool {
56 match self.reader.next() {
57 Some(Ok(batch)) => {
58 let src_array = batch
59 .column_by_name("umi1")
60 .expect("Could not find umi1 column in data")
61 .as_any()
62 .downcast_ref::<PrimitiveArray<UInt64Type>>()
63 .expect("Column umi1 is not UInt64");
64
65 let dst_array = batch
66 .column_by_name("umi2")
67 .expect("Could not find umi2 column in data")
68 .as_any()
69 .downcast_ref::<PrimitiveArray<UInt64Type>>()
70 .expect("Column umi2 is not UInt64");
71
72 self.col_src = Some(src_array.clone());
73 self.col_dst = Some(dst_array.clone());
74
75 self.batch_len = batch.num_rows();
76 self.current_idx = 0;
77 true
78 }
79 _ => false, }
81 }
82}
83
84impl Iterator for ParquetUMIPairIter {
85 type Item = UMIPair;
86
87 fn next(&mut self) -> Option<Self::Item> {
88 if self.current_idx >= self.batch_len && !self.load_next_batch() {
90 return None; }
92
93 let src = self.col_src.as_ref()?.value(self.current_idx);
94 let dst = self.col_dst.as_ref()?.value(self.current_idx);
95
96 self.current_idx += 1;
97
98 Some((src as UMI, dst as UMI))
99 }
100
101 fn size_hint(&self) -> (usize, Option<usize>) {
102 let len = self.expected_size as usize;
103 (len, Some(len))
104 }
105}
106
107impl ExactSizeIterator for ParquetUMIPairIter {}
108
109pub fn write_record_batches_to_path<P: AsRef<Path>, I>(
114 path: P,
115 schema: SchemaRef,
116 record_batches: I,
117 properties: Option<WriterProperties>,
118) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
119where
120 I: Iterator<Item = RecordBatch>,
121{
122 let file = File::create(path)?;
123 let mut writer = ArrowWriter::try_new(file, schema, properties)?;
124
125 for batch in record_batches {
126 writer.write(&batch)?;
127 }
128
129 writer.close()?;
130 Ok(())
131}
132
133pub fn filter_out_crossing_edges_from_edge_list<PIn: AsRef<Path>, POut: AsRef<Path>, T>(
142 input_edgelist_path: &PIn,
143 output_path: &POut,
144 node_partitioning: &T,
145 mapping: &UmiToNodeIndexMapping,
146) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
147where
148 T: NodePartitioning,
149{
150 let file = File::open(input_edgelist_path)?;
151 let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
152 let mut fields = builder
153 .schema()
154 .fields()
155 .iter()
156 .cloned()
157 .collect::<Vec<Arc<Field>>>();
158 let mut reader = builder.with_batch_size(8192).build()?;
159
160 let output_file = File::create(output_path)?;
161 fields.push(Arc::new(Field::new("component", DataType::Utf8, true)));
162 let new_schema = Arc::new(Schema::new(fields));
163 let mut writer = ArrowWriter::try_new(output_file, new_schema.clone(), None)?;
164
165 while let Some(Ok(batch)) = reader.next() {
166 let component1_iter = batch
167 .column_by_name("umi1")
168 .expect("Could not find umi1 column in data")
169 .as_any()
170 .downcast_ref::<PrimitiveArray<UInt64Type>>()
171 .expect("Column umi1 is not UInt64")
172 .iter()
173 .map(|umi| {
174 let umi = umi.expect("umi1 column contains null values");
175 node_partitioning
176 .get_node_to_partition_map()
177 .get(mapping.map_umi_to_node_index(umi as UMI))
178 .unwrap_or_else(|| panic!("umi {} not found in umi mapping", umi))
179 });
180
181 let component2_iter = batch
182 .column_by_name("umi2")
183 .expect("Could not find umi2 column in data")
184 .as_any()
185 .downcast_ref::<PrimitiveArray<UInt64Type>>()
186 .expect("Column umi2 is not UInt64")
187 .iter()
188 .map(|umi| {
189 let umi = umi.expect("umi2 column contains null values");
190 node_partitioning
191 .get_node_to_partition_map()
192 .get(mapping.map_umi_to_node_index(umi as UMI))
193 .unwrap_or_else(|| panic!("umi {} not found in umi mapping", umi))
194 });
195
196 let component_col = StringArray::from(
197 component1_iter
198 .zip(component2_iter)
199 .map(|(c1, c2)| if c1 == c2 { Some(c1.to_string()) } else { None })
200 .collect::<Vec<Option<String>>>(),
201 );
202
203 let mask: BooleanArray = component_col.iter().map(|c| c.is_some()).collect();
204 let mut columns = batch.columns().to_vec();
205 columns.push(Arc::new(component_col));
206 let new_batch = RecordBatch::try_new(new_schema.clone(), columns)?;
207
208 let filtered_batch = compute::filter_record_batch(&new_batch, &mask)?;
209
210 if filtered_batch.num_rows() > 0 {
211 writer.write(&filtered_batch)?;
212 }
213 }
214
215 writer.close()?;
216
217 Ok(())
218}
219
220pub fn write_node_partitions_to_parquet<P: AsRef<Path>, T>(
221 path: P,
222 node_partitioning: &T,
223 mapping: &UmiToNodeIndexMapping,
224 batch_size: Option<usize>,
225) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
226where
227 T: NodePartitioning,
228{
229 let schema = Arc::new(Schema::new(vec![
230 Field::new("umi", DataType::UInt64, false),
231 Field::new("partition_id", DataType::UInt64, false),
232 ]));
233
234 let mapping_node_to_partition = node_partitioning
235 .get_node_to_partition_map()
236 .iter()
237 .enumerate()
238 .map(|(node_idx, partition_idx)| (mapping.map_node_index_to_umi(node_idx), partition_idx));
239
240 let chunk_size = batch_size.unwrap_or(4096);
241 let chunks = mapping_node_to_partition.chunks(chunk_size);
242
243 let record_batches = chunks.into_iter().map(|chunk| {
244 let mut umis: Vec<u64> = Vec::with_capacity(chunk_size);
245 let mut partitions: Vec<u64> = Vec::with_capacity(chunk_size);
246
247 for (umi, partition) in chunk {
248 umis.push(umi as u64);
249 partitions.push(*partition as u64);
250 }
251
252 let umi_array = Arc::new(UInt64Array::from(umis));
253 let partition_array = Arc::new(UInt64Array::from(partitions));
254
255 RecordBatch::try_new(schema.clone(), vec![umi_array, partition_array])
256 .expect("Failed to build record batch")
257 });
258
259 write_record_batches_to_path(path, schema.clone(), record_batches, None)
260}
261
262pub fn create_graph_and_umi_mapping_from_parquet_file<T>(
263 parquet_file: &str,
264) -> (UmiToNodeIndexMapping, Graph<T>)
265where
266 T: EdgeWeight,
267{
268 info!("Creating UMI mapping and graph edges...");
269 let pairs = ParquetUMIPairIter::new(parquet_file).expect("Failed to create ParquetUMIPairIter");
270 let num_pairs = pairs.size_hint().0;
272 let mut umi_mapping = UmiToNodeIndexMapping::with_capacity(num_pairs.saturating_mul(2) / 3);
273 let triplets: Vec<(NodeIdx, NodeIdx, T)> = pairs
274 .map(|(src, dest)| {
275 let (src, dest) = (
276 umi_mapping.get_or_insert(src),
277 umi_mapping.get_or_insert(dest),
278 );
279 (src.min(dest), src.max(dest), T::one())
280 })
281 .collect();
282
283 let num_nodes = umi_mapping.get_num_of_nodes();
284 info!("Creating graph...");
285 let graph = Graph::<T>::from_undirected_triplets(triplets, num_nodes);
286 info!(
287 "Graph created with {} nodes, {} edge entries, total edge weight {}",
288 graph.get_num_nodes(),
289 graph.get_edge_entry_count(),
290 graph.get_total_edge_weight()
291 );
292 (umi_mapping, graph)
293}
294
295#[cfg(test)]
296mod tests {
297 use super::*;
298
299 use crate::common::node_partitioning::FastNodePartitioning;
300 use crate::common::types::PartitionId;
301 use itertools::izip;
302 use parquet::basic::Compression;
303 use tempfile::NamedTempFile;
304
305 #[test]
306 fn test_filter_edge_list() {
307 let test_data = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
308 .join("test_data/mix_40cells_1pc_1000rows.parquet");
309 let (umi_mapping, graph) = create_graph_and_umi_mapping_from_parquet_file::<u8>(
310 test_data
311 .to_str()
312 .expect("File name is not a valid UTF-8 string"),
313 );
314 let partitioning = FastNodePartitioning::initialize_from_partitions(
316 (0..graph.get_num_nodes())
317 .map(|node_id| umi_mapping.map_node_index_to_umi(node_id) % 4)
318 .collect::<Vec<PartitionId>>(),
319 );
320
321 let output_file = NamedTempFile::new().expect("Failed to create tmp file");
322 let temp_file = std::fs::File::open(output_file.path()).unwrap();
323
324 let _ = filter_out_crossing_edges_from_edge_list(
325 &test_data,
326 &output_file,
327 &partitioning,
328 &umi_mapping,
329 );
330
331 let reader_builder = ParquetRecordBatchReaderBuilder::try_new(temp_file).unwrap();
332 let reader = reader_builder.build().unwrap();
333 assert!(
334 reader
335 .flat_map(|batch| {
336 let batch = batch.unwrap();
337 let umi1 = batch
338 .column_by_name("umi1")
339 .unwrap()
340 .as_any()
341 .downcast_ref::<PrimitiveArray<UInt64Type>>()
342 .unwrap()
343 .clone();
344 let umi2 = batch
345 .column_by_name("umi2")
346 .unwrap()
347 .as_any()
348 .downcast_ref::<PrimitiveArray<UInt64Type>>()
349 .unwrap()
350 .clone();
351 let component = batch
352 .column_by_name("component")
353 .unwrap()
354 .as_any()
355 .downcast_ref::<StringArray>()
356 .unwrap()
357 .into_iter()
358 .map(|s| s.unwrap().to_string())
359 .collect::<Vec<String>>();
360
361 izip!(umi1.into_iter(), umi2.into_iter(), component.into_iter()).collect::<Vec<(
362 Option<u64>,
363 Option<u64>,
364 String,
365 )>>(
366 )
367 })
368 .map(|(umi1, umi2, component)| (umi1.unwrap(), umi2.unwrap(), component))
369 .all(
370 |(umi1, umi2, component)| (umi1 % 4).to_string() == component
371 && (umi2 % 4).to_string() == component
372 )
373 );
374 }
375
376 #[test]
377 fn test_write_record_batches_to_path() {
378 let schema = Arc::new(Schema::new(vec![Field::new(
379 "value",
380 DataType::UInt64,
381 false,
382 )]));
383
384 let batches = vec![
385 RecordBatch::try_new(
386 schema.clone(),
387 vec![Arc::new(UInt64Array::from(vec![1, 2, 3]))],
388 )
389 .unwrap(),
390 RecordBatch::try_new(
391 schema.clone(),
392 vec![Arc::new(UInt64Array::from(vec![4, 5]))],
393 )
394 .unwrap(),
395 ];
396
397 let output_file = NamedTempFile::new().expect("Failed to create tmp file");
398
399 write_record_batches_to_path(
400 output_file.path(),
401 schema.clone(),
402 batches.into_iter(),
403 None,
404 )
405 .expect("Failed to write record batches");
406
407 let temp_file = std::fs::File::open(output_file.path()).unwrap();
408 let reader = ParquetRecordBatchReaderBuilder::try_new(temp_file)
409 .unwrap()
410 .build()
411 .unwrap();
412
413 let values = reader
414 .flat_map(|batch| {
415 batch
416 .unwrap()
417 .column_by_name("value")
418 .unwrap()
419 .as_any()
420 .downcast_ref::<UInt64Array>()
421 .unwrap()
422 .iter()
423 .map(|v| v.unwrap())
424 .collect::<Vec<u64>>()
425 })
426 .collect::<Vec<u64>>();
427
428 assert_eq!(values, vec![1, 2, 3, 4, 5]);
429 }
430
431 #[test]
432 fn test_write_record_batches_to_path_uses_properties() {
433 let schema = Arc::new(Schema::new(vec![Field::new(
434 "value",
435 DataType::UInt64,
436 false,
437 )]));
438
439 let batches = vec![
440 RecordBatch::try_new(
441 schema.clone(),
442 vec![Arc::new(UInt64Array::from(vec![1, 2, 3]))],
443 )
444 .unwrap(),
445 ];
446
447 let properties = WriterProperties::builder()
448 .set_compression(Compression::SNAPPY)
449 .build();
450
451 let output_file = NamedTempFile::new().expect("Failed to create tmp file");
452
453 write_record_batches_to_path(
454 output_file.path(),
455 schema.clone(),
456 batches.into_iter(),
457 Some(properties),
458 )
459 .expect("Failed to write record batches");
460
461 let temp_file = std::fs::File::open(output_file.path()).unwrap();
462 let builder = ParquetRecordBatchReaderBuilder::try_new(temp_file).unwrap();
463
464 assert_eq!(
465 builder.metadata().row_group(0).column(0).compression(),
466 Compression::SNAPPY,
467 );
468 }
469}