diff --git a/.gitignore b/.gitignore index 7c6e94c..2f098ad 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,5 @@ *.db dfs_skel/ venv/ -dn1/ -dn2/ +dn*/ copy_dir/ diff --git a/Makefile b/Makefile index 6b58a6c..df58831 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,19 @@ all: build -dist: build - @rm -f assig-03-dfs.tar.gz - @cp target/release/copy copy - @tar -czf assig-03-dfs.tar.gz src/ README.md copy +dist: clean build + @cp target/release/copy . + @cp target/release/ls . + @cp target/release/data_node . + @cp target/release/meta_data . + @tar -czf assig-03-dfs.tar.gz src/ README.md copy ls data_node meta_data Cargo.* clean_db createdb.py @rm copy + @rm ls + @rm data_node + @rm meta_data + +clean: + @rm -f assig-03-dfs.tar.gz + build: cargo build --release diff --git a/README.md b/README.md index c557304..697ffa2 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,85 @@ ## Distributed File System in Rust for CCOM4017 -#### Running +This suite of programs handles file copying over TCP with a client/server model. +It contains the following programs; +- copy +- ls +- data_node +- meta_data -```./copy ``` +`copy` and `ls` are clients that connect to the servers. `copy` sends file read and write requests +to the `meta_data` server, which uses a sqlite3 database to keep track of which nodes are connected, +as well as which files have been added. When a file is added, `meta_data` sends the list of available +`data_node` servers, `copy` then divides the file up by the amount of nodes, then proceeds to transfer +each chunk over 256 bytes at a time. `ls` simply prints out a list of the existing files on the +`meta_data` server. -#### Building +The code uses `serde_json` to serialize and deserialize Rust structs to and from json. The clients and +servers then listen for incoming streams of data and parses them as json. As well as exchanging +metadata, this protocol also establishes the handshake to then transfer the raw file chunks. + +`rusqlite` is used for managing the sqlite database. This allows SQL queries to be performed from +the rust code and manage the data base in a relatively type safe way. Unit tests in the `meta_data` +provide coverage of these SQL operations against an in-memory version + +### WARNING: +If you're my professor, please do not generate a database with the default `createdb.py` +provided in the skeleton dfs. I have included a custom version of the file in the root of the project. +The reason being that I changed chunks to be integers rather than strings, in order to provide ordering +to the chunks when transferring. + +##### Running + +To run the `ls` provide an endpoint in the _`ip:port`_ format. _`ip`_ can be _"localhost"_, consider +using `./` to avoid a naming conflict with the GNU version of `ls` + +```$ ./ls 127.0.0.1:6770``` + +The `meta_data` server takes an optional port, but will default to `8000` if none is specified. + +```$ meta_data 6710``` + +The data node takes two endpoints in the _`ip:port`_ and then a an optional path. The first endpoint +is the ip and port, both for binding to a TCP port and also to send itself to the `meta_data` server. +The second endpoint is the `meta_data` server's ip and port. The optional base path will default to the +working directory if none is provided. + +```$ data_node localhost:6771 127.0.0.1:8000 my_cool_data_node``` + +The `copy` takes two different parameter versions, depending on whether it's sending to or receiving +from the server. To send a file, provide the path to the local file, then the endpoint with the file +in the _`ip:host:filepath`_ format. The `data_node` will save the file relative to the base path +provided to it. + +```$ copy some_path/pug.jpg localhost:6700:another_path/pug.jpg``` + +To receive a file, simply invert the parameters + +```$ copy localhost:6700:another_path/pug.jpg some_path/pug.jpg``` + +##### Misc Scripts + +`shutdown_node` sends a json request with a provided port to shutdown a `data_node`. This ensures +that the node can terminate gracefully and unregister itself from the `meta_data` server. I was +advised against using Unix Signals, so opted for this instead. + +```$ shutdown_node 6770``` + +`sm` just does a _send message_ to a provide port. It can be used to test and inspect jsons. It can +for instance be used to mimic the `ls`; + +``` +$ sm '{"p_type":"ListFiles","json":null}' 8000 +Connection to localhost 8000 port [tcp/*] succeeded! +{"paths":["pug.jpg 21633 bytes"]}% +``` + +`clean_db` just recreates the `dfs.db` with the custom python script. + +##### Building If you wish to compile the code, install rust and cargo -[Link](https://www.rust-lang.org/en-US/install.html) +[link](https://www.rust-lang.org/en-US/install.html) Then just run build @@ -17,5 +89,6 @@ If you wish to run a specific algorithm; ```cargo run --bin copy ``` -#### Testing +##### Testing +`cargo test --bin meta_data` diff --git a/sm b/sm index ed7592f..48f3f71 100755 --- a/sm +++ b/sm @@ -1 +1 @@ -echo $1 | nc -v -N localhost 6770 +echo $1 | nc -v -N localhost $2 diff --git a/src/bin/data_node.rs b/src/bin/data_node.rs index 3890e75..900024f 100644 --- a/src/bin/data_node.rs +++ b/src/bin/data_node.rs @@ -1,25 +1,23 @@ extern crate a03; extern crate serde; extern crate serde_json; -#[macro_use] extern crate serde_derive; use a03::*; use std::net::{TcpStream, Shutdown}; -use std::io::{Write, Read}; +use std::io::{Write, BufWriter}; use std::net::TcpListener; use serde_json::from_str; use std::fs::File; use std::fs; use std::error::Error; -use std::io::BufWriter; use std::time::Instant; fn main() { let node_endpoint = parse_endpoint_from_cli(0); let metadata_endpoint = parse_endpoint_from_cli(1); let data_path = std::env::args().skip(3).next() - .expect("Missing data path"); + .unwrap_or(String::from(".")); let listener = TcpListener::bind(&node_endpoint).unwrap(); register_with_meta_server(&metadata_endpoint, &node_endpoint); diff --git a/src/bin/ls.rs b/src/bin/ls.rs index 8aa8434..4038ded 100644 --- a/src/bin/ls.rs +++ b/src/bin/ls.rs @@ -1,14 +1,10 @@ extern crate a03; extern crate serde; extern crate serde_json; -#[macro_use] extern crate serde_derive; use a03::*; -use std::net::{TcpListener, TcpStream, Shutdown, SocketAddrV4, Ipv4Addr}; -use std::borrow::Cow; -use std::thread; -use std::io::Read; +use std::net::{TcpStream, Shutdown }; use std::io::Write; fn main() { diff --git a/src/lib.rs b/src/lib.rs index de4015b..c5e0708 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,7 +91,7 @@ pub struct BlockQuery { } pub fn parse_endpoint_from_cli(arg_index : usize) -> String { - let mut args: Vec = std::env::args().skip(1).collect(); + let args: Vec = std::env::args().skip(1).collect(); let endpoint_arg: String = args.get(arg_index).expect("No IP provided").clone(); if endpoint_arg.contains(":") {