Interactive Analytics at Scale: BlinkDB, VerdictDB, and SnappyData
Systems for interactive analytics over massive data with explicit guarantees on accuracy and response time.
Research question
Can large-scale analytics become interactive while making accuracy and latency tradeoffs explicit?
Approach
Use sampling, error estimation, stochastic planning, and platform-independent execution to expose bounded tradeoffs and reusable analytics systems.
Commercialization and acquisition
BlinkDB was commercialized through SnappyData, which was later acquired by TIBCO.
Key results
- BlinkDB uses adaptive, multi-dimensional sampling to answer interactive SQL queries with explicit error or response-time bounds.
- VerdictDB brings approximate query processing to existing SQL engines without requiring a new database.
- BlinkDB was commercialized through SnappyData, which was later acquired by TIBCO.
Publications
BlinkDB: queries with bounded errors and bounded response times on very large data
EuroSys 2013 · ACM European Conference on Computer Systems
Cite
@inproceedings{7c357390-71f0-46af-b841-6dd64ddd5193,
title = {BlinkDB: queries with bounded errors and bounded response times on very large data},
author = {Sameer Agarwal and Barzan Mozafari and Aurojit Panda and Henry Milner and Samuel Madden and Ion Stoica},
booktitle = {ACM European Conference on Computer Systems},
year = {2013},
doi = {10.1145/2465351.2465355}
}The analytical bootstrap: a new method for fast error estimation in approximate query processing
SIGMOD 2014 · ACM SIGMOD International Conference on Management of Data
Cite
@inproceedings{2120f282-9549-4542-91fb-e9d4cf1d6c2f,
title = {The analytical bootstrap: a new method for fast error estimation in approximate query processing},
author = {Kai Zeng and Shi Gao and Barzan Mozafari and Carlo Zaniolo},
booktitle = {ACM SIGMOD International Conference on Management of Data},
year = {2014},
doi = {10.1145/2588555.2588579}
}Verdict: A System for Stochastic Query Planning
CIDR 2015 · Conference on Innovative Data Systems Research
VerdictDB: Universalizing Approximate Query Processing
SIGMOD 2018 · ACM SIGMOD International Conference on Management of Data
Cite
@inproceedings{ed762d26-878b-453f-b272-09d069f37d64,
title = {VerdictDB: Universalizing Approximate Query Processing},
author = {Yongjoo Park and Barzan Mozafari and Joseph Sorenson and Junhao Wang},
booktitle = {ACM SIGMOD International Conference on Management of Data},
year = {2018},
doi = {10.1145/3183713.3196905}
}Join on Samples: A Theoretical Guide for Practitioners
PVLDB 2019 · Proceedings of the VLDB Endowment
Cite
@article{6d32861d-ba95-4e8b-91c3-bc689ec60e12,
title = {Join on Samples: A Theoretical Guide for Practitioners},
author = {Dawei Huang and Dong Young Yoon and Seth Pettie and Barzan Mozafari},
journal = {Proceedings of the VLDB Endowment},
year = {2019},
doi = {10.14778/3372716.3372726}
}SnappyData
Encyclopedia of Big Data Technologies 2019
Demonstration of VerdictDB, the Platform-Independent AQP System
SIGMOD 2018 · ACM SIGMOD International Conference on Management of Data
Cite
@inproceedings{9438c730-b302-49f6-bed8-3a77c1e7a2c2,
title = {Demonstration of VerdictDB, the Platform-Independent AQP System},
author = {Wen He and Yongjoo Park and Idris Hanafi and Jacob Yatvitskiy and Barzan Mozafari},
booktitle = {ACM SIGMOD International Conference on Management of Data},
year = {2018},
doi = {10.1145/3183713.3193538}
}Approximate Query Engines: Commercial Challenges and Research Opportunities
SIGMOD 2017 · ACM SIGMOD International Conference on Management of Data
Cite
@inproceedings{fc569ad9-4672-4854-9fef-d8bd5b3ec51c,
title = {Approximate Query Engines: Commercial Challenges and Research Opportunities},
author = {Barzan Mozafari},
booktitle = {ACM SIGMOD International Conference on Management of Data},
year = {2017},
doi = {10.1145/3035918.3056098}
}SnappyData: A Unified Cluster for Streaming, Transactions and Interactive Analytics
CIDR 2017 · Conference on Innovative Data Systems Research
Cite
@inproceedings{e6de4a52-07a5-47c5-8196-ace46d2a49c2,
title = {SnappyData: A Unified Cluster for Streaming, Transactions and Interactive Analytics},
author = {Barzan Mozafari and Jags Ramnarayan and Sudhir Menon and Yogesh Mahajan and Soubhik Chakraborty and Hemant Bhanawat and Kishor Bachhav},
booktitle = {Conference on Innovative Data Systems Research},
year = {2017}
}SnappyData: A Hybrid Transactional Analytical Store Built On Spark
SIGMOD 2016 · ACM SIGMOD International Conference on Management of Data
Cite
@inproceedings{cea59a03-38c4-4acb-bd2d-a41ecabe154a,
title = {SnappyData: A Hybrid Transactional Analytical Store Built On Spark},
author = {Jags Ramnarayan and Barzan Mozafari and Sumedh Wale and Sudhir Menon and Neeraj Kumar and Hemant Bhanawat and Soubhik Chakraborty and Yogesh Mahajan and Rishitesh Mishra and Kishor Bachhav},
booktitle = {ACM SIGMOD International Conference on Management of Data},
year = {2016},
doi = {10.1145/2882903.2899408}
}Visualization-aware sampling for very large databases
ICDE 2016 · IEEE International Conference on Data Engineering
Cite
@inproceedings{fb040b81-82bc-493d-9322-fd576b39c635,
title = {Visualization-aware sampling for very large databases},
author = {Yongjoo Park and Michael J. Cafarella and Barzan Mozafari},
booktitle = {IEEE International Conference on Data Engineering},
year = {2016},
doi = {10.1109/ICDE.2016.7498287}
}A Handbook for Building an Approximate Query Engine
IEEE Data Engineering Bulletin 2015
Knowing when you're wrong: building fast and reliable approximate query processing systems
SIGMOD 2014 · ACM SIGMOD International Conference on Management of Data
Cite
@inproceedings{40bace5f-2a47-4fe3-b90a-bd42d385640b,
title = {Knowing when you're wrong: building fast and reliable approximate query processing systems},
author = {Sameer Agarwal and Henry Milner and Ariel Kleiner and Ameet Talwalkar and Michael I. Jordan and Samuel Madden and Barzan Mozafari and Ion Stoica},
booktitle = {ACM SIGMOD International Conference on Management of Data},
year = {2014},
doi = {10.1145/2588555.2593667}
}