diff --git a/README.md b/README.md index b444c46..80ad9fe 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,47 @@ -# webObserver -Observes selected websites using cURL and observe changes +# Web Observer + +The application is made to observe selected websites using cURL and calculate the changes using ssdeep fuzzy hash. Should the changes exceed selected treshold, the application would record the diff in a file + +## Getting Started + +After installation, you may insert the websites you want to monitor in 'curlfile.txt'. The format goes like this: +``` +Example Site 1 +curl https://www.example.com +Example Site 2 +curl https://www.example2.com +Example Site 3 +curl https://www.example3.com +``` + +### Prerequisites + +The application uses gohtml to beautify html and ssdeep to conduct fuzzy hashing. + +``` +go get github.com/yosssi/gohtml +go get github.com/glaslos/ssdeep +``` + +### Installing + +A step by step series of examples that tell you how to get a development env running + +Say what the step will be + +``` +Give the example +``` + +### TODOs +- Improve the current shitty JS beautify +- Include functionality to email a user when changes are observed + +## Authors + +* **Samuel Pua** - *Initial work* - [Github](https://github.com/telboon) + +## License + +This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details + diff --git a/curlfile.txt b/curlfile.txt new file mode 100644 index 0000000..ea935f4 --- /dev/null +++ b/curlfile.txt @@ -0,0 +1,6 @@ +Google +curl https://www.google.com +Samuel +curl https://www.samuelpua.com/ +jstest +curl https://www.samuelpua.com/wp-includes/js/jquery/jquery.js diff --git a/main.go b/main.go new file mode 100644 index 0000000..652b358 --- /dev/null +++ b/main.go @@ -0,0 +1,117 @@ +package main + +import ( + "os/exec" + "os" + "io/ioutil" + "fmt" + "strings" + "bytes" + "github.com/yosssi/gohtml" + "github.com/glaslos/ssdeep" + "time" +) + +const ( + curlfile = "./curlfile.txt" + resultpath = "./results/" + diffpath = "./diff/" + hashTreshold = 90 +) + +func main() { + var scoreOutput string + + fullBytes, _ := ioutil.ReadFile(curlfile) + fullStr := string(fullBytes) + siteList := make([]string, 0) + siteList = strings.Split(fullStr, "\n") + + siteNo := len(siteList)/2 + + for i:=0; i