Skip to main content

pixelator_core/common/
io.rs

1use std::fs::File;
2use std::path::Path;
3use std::sync::Arc;
4
5use arrow::array::StringArray;
6use arrow::array::{Array, BooleanArray, PrimitiveArray, RecordBatch, UInt64Array};
7use arrow::compute;
8use arrow::datatypes::{DataType, Field, Schema, SchemaRef, UInt64Type};
9use itertools::Itertools;
10use log::info;
11use parquet::arrow::ArrowWriter;
12use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
13use parquet::file::properties::WriterProperties;
14
15use crate::common::graph::Graph;
16use crate::common::node_indexing::UmiToNodeIndexMapping;
17use crate::common::node_partitioning::NodePartitioning;
18use crate::common::types::{EdgeWeight, NodeIdx, UMI, UMIPair};
19
20/// An iterator that yields UMI pairs from a parquet file one by one.
21/// These can be mapped to node indices and used to construct a graph.
22pub struct ParquetUMIPairIter {
23    reader: ParquetRecordBatchReader,
24
25    expected_size: i64,
26
27    // Buffers for the currently loaded batch
28    // We hold the specific typed arrays to avoid downcasting on every row
29    col_src: Option<PrimitiveArray<UInt64Type>>,
30    col_dst: Option<PrimitiveArray<UInt64Type>>,
31
32    // Pointers for iteration logic
33    current_idx: usize,
34    batch_len: usize,
35}
36
37impl ParquetUMIPairIter {
38    pub fn new<P: AsRef<Path>>(path: P) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
39        let file = File::open(path)?;
40
41        let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
42        let num_rows = builder.metadata().file_metadata().num_rows();
43        let reader = builder.with_batch_size(8192).build()?;
44
45        Ok(Self {
46            reader,
47            expected_size: num_rows,
48            col_src: None,
49            col_dst: None,
50            current_idx: 0,
51            batch_len: 0,
52        })
53    }
54
55    fn load_next_batch(&mut self) -> bool {
56        match self.reader.next() {
57            Some(Ok(batch)) => {
58                let src_array = batch
59                    .column_by_name("umi1")
60                    .expect("Could not find umi1 column in data")
61                    .as_any()
62                    .downcast_ref::<PrimitiveArray<UInt64Type>>()
63                    .expect("Column umi1 is not UInt64");
64
65                let dst_array = batch
66                    .column_by_name("umi2")
67                    .expect("Could not find umi2 column in data")
68                    .as_any()
69                    .downcast_ref::<PrimitiveArray<UInt64Type>>()
70                    .expect("Column umi2 is not UInt64");
71
72                self.col_src = Some(src_array.clone());
73                self.col_dst = Some(dst_array.clone());
74
75                self.batch_len = batch.num_rows();
76                self.current_idx = 0;
77                true
78            }
79            _ => false, // End of file or error
80        }
81    }
82}
83
84impl Iterator for ParquetUMIPairIter {
85    type Item = UMIPair;
86
87    fn next(&mut self) -> Option<Self::Item> {
88        // If we are at the end of the current batch, load the next one
89        if self.current_idx >= self.batch_len && !self.load_next_batch() {
90            return None; // No more batches
91        }
92
93        let src = self.col_src.as_ref()?.value(self.current_idx);
94        let dst = self.col_dst.as_ref()?.value(self.current_idx);
95
96        self.current_idx += 1;
97
98        Some((src as UMI, dst as UMI))
99    }
100
101    fn size_hint(&self) -> (usize, Option<usize>) {
102        let len = self.expected_size as usize;
103        (len, Some(len))
104    }
105}
106
107impl ExactSizeIterator for ParquetUMIPairIter {}
108
109/// Write an iterator of record batches to a parquet file at the given path.
110///
111/// The `properties` argument configures the underlying parquet writer (e.g.
112/// compression); pass `None` to use the writer's defaults.
113pub fn write_record_batches_to_path<P: AsRef<Path>, I>(
114    path: P,
115    schema: SchemaRef,
116    record_batches: I,
117    properties: Option<WriterProperties>,
118) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
119where
120    I: Iterator<Item = RecordBatch>,
121{
122    let file = File::create(path)?;
123    let mut writer = ArrowWriter::try_new(file, schema, properties)?;
124
125    for batch in record_batches {
126        writer.write(&batch)?;
127    }
128
129    writer.close()?;
130    Ok(())
131}
132
133/// Remove crossing edges from the provided edge list and write the result to
134/// a new parquet file.
135///
136/// # Arguments
137/// * `input_edgelist_path`: path to the edge list to filtler
138/// * `output_path`: path where the resulting filtered edgelist will be written
139/// * `node_partitioning`: partitioning to be used for detecting crossing edges
140/// * `mapping`: mapping between umis and node indices
141pub fn filter_out_crossing_edges_from_edge_list<PIn: AsRef<Path>, POut: AsRef<Path>, T>(
142    input_edgelist_path: &PIn,
143    output_path: &POut,
144    node_partitioning: &T,
145    mapping: &UmiToNodeIndexMapping,
146) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
147where
148    T: NodePartitioning,
149{
150    let file = File::open(input_edgelist_path)?;
151    let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
152    let mut fields = builder
153        .schema()
154        .fields()
155        .iter()
156        .cloned()
157        .collect::<Vec<Arc<Field>>>();
158    let mut reader = builder.with_batch_size(8192).build()?;
159
160    let output_file = File::create(output_path)?;
161    fields.push(Arc::new(Field::new("component", DataType::Utf8, true)));
162    let new_schema = Arc::new(Schema::new(fields));
163    let mut writer = ArrowWriter::try_new(output_file, new_schema.clone(), None)?;
164
165    while let Some(Ok(batch)) = reader.next() {
166        let component1_iter = batch
167            .column_by_name("umi1")
168            .expect("Could not find umi1 column in data")
169            .as_any()
170            .downcast_ref::<PrimitiveArray<UInt64Type>>()
171            .expect("Column umi1 is not UInt64")
172            .iter()
173            .map(|umi| {
174                let umi = umi.expect("umi1 column contains null values");
175                node_partitioning
176                    .get_node_to_partition_map()
177                    .get(mapping.map_umi_to_node_index(umi as UMI))
178                    .unwrap_or_else(|| panic!("umi {} not found in umi mapping", umi))
179            });
180
181        let component2_iter = batch
182            .column_by_name("umi2")
183            .expect("Could not find umi2 column in data")
184            .as_any()
185            .downcast_ref::<PrimitiveArray<UInt64Type>>()
186            .expect("Column umi2 is not UInt64")
187            .iter()
188            .map(|umi| {
189                let umi = umi.expect("umi2 column contains null values");
190                node_partitioning
191                    .get_node_to_partition_map()
192                    .get(mapping.map_umi_to_node_index(umi as UMI))
193                    .unwrap_or_else(|| panic!("umi {} not found in umi mapping", umi))
194            });
195
196        let component_col = StringArray::from(
197            component1_iter
198                .zip(component2_iter)
199                .map(|(c1, c2)| if c1 == c2 { Some(c1.to_string()) } else { None })
200                .collect::<Vec<Option<String>>>(),
201        );
202
203        let mask: BooleanArray = component_col.iter().map(|c| c.is_some()).collect();
204        let mut columns = batch.columns().to_vec();
205        columns.push(Arc::new(component_col));
206        let new_batch = RecordBatch::try_new(new_schema.clone(), columns)?;
207
208        let filtered_batch = compute::filter_record_batch(&new_batch, &mask)?;
209
210        if filtered_batch.num_rows() > 0 {
211            writer.write(&filtered_batch)?;
212        }
213    }
214
215    writer.close()?;
216
217    Ok(())
218}
219
220pub fn write_node_partitions_to_parquet<P: AsRef<Path>, T>(
221    path: P,
222    node_partitioning: &T,
223    mapping: &UmiToNodeIndexMapping,
224    batch_size: Option<usize>,
225) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
226where
227    T: NodePartitioning,
228{
229    let schema = Arc::new(Schema::new(vec![
230        Field::new("umi", DataType::UInt64, false),
231        Field::new("partition_id", DataType::UInt64, false),
232    ]));
233
234    let mapping_node_to_partition = node_partitioning
235        .get_node_to_partition_map()
236        .iter()
237        .enumerate()
238        .map(|(node_idx, partition_idx)| (mapping.map_node_index_to_umi(node_idx), partition_idx));
239
240    let chunk_size = batch_size.unwrap_or(4096);
241    let chunks = mapping_node_to_partition.chunks(chunk_size);
242
243    let record_batches = chunks.into_iter().map(|chunk| {
244        let mut umis: Vec<u64> = Vec::with_capacity(chunk_size);
245        let mut partitions: Vec<u64> = Vec::with_capacity(chunk_size);
246
247        for (umi, partition) in chunk {
248            umis.push(umi as u64);
249            partitions.push(*partition as u64);
250        }
251
252        let umi_array = Arc::new(UInt64Array::from(umis));
253        let partition_array = Arc::new(UInt64Array::from(partitions));
254
255        RecordBatch::try_new(schema.clone(), vec![umi_array, partition_array])
256            .expect("Failed to build record batch")
257    });
258
259    write_record_batches_to_path(path, schema.clone(), record_batches, None)
260}
261
262pub fn create_graph_and_umi_mapping_from_parquet_file<T>(
263    parquet_file: &str,
264) -> (UmiToNodeIndexMapping, Graph<T>)
265where
266    T: EdgeWeight,
267{
268    info!("Creating UMI mapping and graph edges...");
269    let pairs = ParquetUMIPairIter::new(parquet_file).expect("Failed to create ParquetUMIPairIter");
270    // Mean degree about 3, so the node count is near 2E/3.
271    let num_pairs = pairs.size_hint().0;
272    let mut umi_mapping = UmiToNodeIndexMapping::with_capacity(num_pairs.saturating_mul(2) / 3);
273    let triplets: Vec<(NodeIdx, NodeIdx, T)> = pairs
274        .map(|(src, dest)| {
275            let (src, dest) = (
276                umi_mapping.get_or_insert(src),
277                umi_mapping.get_or_insert(dest),
278            );
279            (src.min(dest), src.max(dest), T::one())
280        })
281        .collect();
282
283    let num_nodes = umi_mapping.get_num_of_nodes();
284    info!("Creating graph...");
285    let graph = Graph::<T>::from_undirected_triplets(triplets, num_nodes);
286    info!(
287        "Graph created with {} nodes, {} edge entries, total edge weight {}",
288        graph.get_num_nodes(),
289        graph.get_edge_entry_count(),
290        graph.get_total_edge_weight()
291    );
292    (umi_mapping, graph)
293}
294
295#[cfg(test)]
296mod tests {
297    use super::*;
298
299    use crate::common::node_partitioning::FastNodePartitioning;
300    use crate::common::types::PartitionId;
301    use itertools::izip;
302    use parquet::basic::Compression;
303    use tempfile::NamedTempFile;
304
305    #[test]
306    fn test_filter_edge_list() {
307        let test_data = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
308            .join("test_data/mix_40cells_1pc_1000rows.parquet");
309        let (umi_mapping, graph) = create_graph_and_umi_mapping_from_parquet_file::<u8>(
310            test_data
311                .to_str()
312                .expect("File name is not a valid UTF-8 string"),
313        );
314        // Make up partitions based on the mod 4 parity of umis
315        let partitioning = FastNodePartitioning::initialize_from_partitions(
316            (0..graph.get_num_nodes())
317                .map(|node_id| umi_mapping.map_node_index_to_umi(node_id) % 4)
318                .collect::<Vec<PartitionId>>(),
319        );
320
321        let output_file = NamedTempFile::new().expect("Failed to create tmp file");
322        let temp_file = std::fs::File::open(output_file.path()).unwrap();
323
324        let _ = filter_out_crossing_edges_from_edge_list(
325            &test_data,
326            &output_file,
327            &partitioning,
328            &umi_mapping,
329        );
330
331        let reader_builder = ParquetRecordBatchReaderBuilder::try_new(temp_file).unwrap();
332        let reader = reader_builder.build().unwrap();
333        assert!(
334            reader
335                .flat_map(|batch| {
336                    let batch = batch.unwrap();
337                    let umi1 = batch
338                        .column_by_name("umi1")
339                        .unwrap()
340                        .as_any()
341                        .downcast_ref::<PrimitiveArray<UInt64Type>>()
342                        .unwrap()
343                        .clone();
344                    let umi2 = batch
345                        .column_by_name("umi2")
346                        .unwrap()
347                        .as_any()
348                        .downcast_ref::<PrimitiveArray<UInt64Type>>()
349                        .unwrap()
350                        .clone();
351                    let component = batch
352                        .column_by_name("component")
353                        .unwrap()
354                        .as_any()
355                        .downcast_ref::<StringArray>()
356                        .unwrap()
357                        .into_iter()
358                        .map(|s| s.unwrap().to_string())
359                        .collect::<Vec<String>>();
360
361                    izip!(umi1.into_iter(), umi2.into_iter(), component.into_iter()).collect::<Vec<(
362                        Option<u64>,
363                        Option<u64>,
364                        String,
365                    )>>(
366                    )
367                })
368                .map(|(umi1, umi2, component)| (umi1.unwrap(), umi2.unwrap(), component))
369                .all(
370                    |(umi1, umi2, component)| (umi1 % 4).to_string() == component
371                        && (umi2 % 4).to_string() == component
372                )
373        );
374    }
375
376    #[test]
377    fn test_write_record_batches_to_path() {
378        let schema = Arc::new(Schema::new(vec![Field::new(
379            "value",
380            DataType::UInt64,
381            false,
382        )]));
383
384        let batches = vec![
385            RecordBatch::try_new(
386                schema.clone(),
387                vec![Arc::new(UInt64Array::from(vec![1, 2, 3]))],
388            )
389            .unwrap(),
390            RecordBatch::try_new(
391                schema.clone(),
392                vec![Arc::new(UInt64Array::from(vec![4, 5]))],
393            )
394            .unwrap(),
395        ];
396
397        let output_file = NamedTempFile::new().expect("Failed to create tmp file");
398
399        write_record_batches_to_path(
400            output_file.path(),
401            schema.clone(),
402            batches.into_iter(),
403            None,
404        )
405        .expect("Failed to write record batches");
406
407        let temp_file = std::fs::File::open(output_file.path()).unwrap();
408        let reader = ParquetRecordBatchReaderBuilder::try_new(temp_file)
409            .unwrap()
410            .build()
411            .unwrap();
412
413        let values = reader
414            .flat_map(|batch| {
415                batch
416                    .unwrap()
417                    .column_by_name("value")
418                    .unwrap()
419                    .as_any()
420                    .downcast_ref::<UInt64Array>()
421                    .unwrap()
422                    .iter()
423                    .map(|v| v.unwrap())
424                    .collect::<Vec<u64>>()
425            })
426            .collect::<Vec<u64>>();
427
428        assert_eq!(values, vec![1, 2, 3, 4, 5]);
429    }
430
431    #[test]
432    fn test_write_record_batches_to_path_uses_properties() {
433        let schema = Arc::new(Schema::new(vec![Field::new(
434            "value",
435            DataType::UInt64,
436            false,
437        )]));
438
439        let batches = vec![
440            RecordBatch::try_new(
441                schema.clone(),
442                vec![Arc::new(UInt64Array::from(vec![1, 2, 3]))],
443            )
444            .unwrap(),
445        ];
446
447        let properties = WriterProperties::builder()
448            .set_compression(Compression::SNAPPY)
449            .build();
450
451        let output_file = NamedTempFile::new().expect("Failed to create tmp file");
452
453        write_record_batches_to_path(
454            output_file.path(),
455            schema.clone(),
456            batches.into_iter(),
457            Some(properties),
458        )
459        .expect("Failed to write record batches");
460
461        let temp_file = std::fs::File::open(output_file.path()).unwrap();
462        let builder = ParquetRecordBatchReaderBuilder::try_new(temp_file).unwrap();
463
464        assert_eq!(
465            builder.metadata().row_group(0).column(0).compression(),
466            Compression::SNAPPY,
467        );
468    }
469}