package ktmtrainbot import ( "context" "errors" "fmt" "log" "os" "strconv" "strings" "time" "git.samuelpua.com/telboon/ktm-train-bot/backend/internal/captchasolver" "git.samuelpua.com/telboon/ktm-train-bot/backend/internal/user" "github.com/go-rod/rod" "github.com/go-rod/rod/lib/devices" "github.com/go-rod/rod/lib/launcher" "github.com/google/uuid" "gorm.io/gorm" "gorm.io/gorm/logger" ) const IMAGE_DIR = "/tmp/screenshots" const TIMEOUT_MINUTE = 60 func (env *Env) BackgroundJobRunner() { log.Println("Initialising background job...") initialiseRodBrowser() log.Println("Browser initialised...") // Initialise silent logger newLogger := logger.New( log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer logger.Config{ SlowThreshold: time.Second, // Slow SQL threshold LogLevel: logger.Silent, // Log level IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger Colorful: true, // Disable color }, ) forceStartBookingString := os.Getenv("FORCE_START_BOOKING") forceStartBooking := false if strings.ToUpper(forceStartBookingString) == "TRUE" { forceStartBooking = true } tx := env.DB.Session(&gorm.Session{Logger: newLogger}) for { var usersWithRunningTask []user.User tx.Table("bookings"). Select("users.*"). Joins("left join users on users.id = bookings.user_id"). Where("bookings.status = ?", "running"). Find(&usersWithRunningTask) var userIDArr []uuid.UUID for _, currUser := range usersWithRunningTask { userIDArr = append(userIDArr, currUser.ID) } var jobToDo Booking var err error if len(userIDArr) > 0 { err = tx.Model(&jobToDo). Where("user_id NOT IN (?)", userIDArr). Where("status = ?", "pending"). Order("created_at asc"). Preload("User"). First(&jobToDo).Error } else { err = tx.Model(&jobToDo). Where("status = ?", "pending"). Order("created_at asc"). Preload("User"). First(&jobToDo).Error } // if no jobs pending found if err != nil { time.Sleep(time.Second) continue } else { // if there's job to do // Create next run where it's not the past (either from old NextRun or now()) timeNow := time.Now() startTime := time.Date(timeNow.Year(), timeNow.Month(), timeNow.Day(), 00, 10, 0, 0, timeNow.Location()) endTime := startTime.Add(15 * time.Minute) if forceStartBooking || (timeNow.After(startTime) && timeNow.Before(endTime)) { err := env.DB.Where(&user.Profile{UserID: jobToDo.UserID}).First(&jobToDo.User.Profile).Error if err != nil { log.Println(err) } log.Printf("Start doing job: %v", jobToDo.ID) username := jobToDo.User.Profile.KtmTrainUsername password := jobToDo.User.Profile.KtmTrainPassword creditCardType := jobToDo.User.Profile.KtmTrainCreditCardType creditCard := jobToDo.User.Profile.KtmTrainCreditCard creditCardCVV := jobToDo.User.Profile.KtmTrainCreditCardCVV creditCardExpiry := jobToDo.User.Profile.KtmTrainCreditCardExpiry go func() { defer func() { if r := recover(); r != nil { log.Println("Recovering from job panic...") jobToDo.Status = "pending" env.DB.Save(&jobToDo) } }() success := env.startBooking(&jobToDo, username, password, creditCardType, creditCard, creditCardCVV, creditCardExpiry) if success { jobToDo.Status = "success" env.DB.Save(&jobToDo) log.Println("Successfully made a booking.") } else { // Sleep for 10 minutes before setting status to pending time.Sleep(10 * time.Minute) jobToDo.Status = "pending" env.DB.Save(&jobToDo) log.Println("Failed to make a booking.") } }() jobToDo.Status = "running" env.DB.Save(&jobToDo) log.Printf("Job Started: %v", jobToDo.ID) } else { time.Sleep(time.Second) } } } } func initialiseRodBrowser() { u := launcher.New(). Set("headless"). MustLaunch() defaultDevice := devices.LaptopWithMDPIScreen browser := rod.New().ControlURL(u).MustConnect().DefaultDevice(defaultDevice) page := browser.MustPage("https://www.google.com").MustWindowFullscreen() page.MustWaitLoad() browser.MustClose() // Initialise screenshot directory if _, err := os.Stat(IMAGE_DIR); errors.Is(err, os.ErrNotExist) { err := os.Mkdir(IMAGE_DIR, os.ModePerm) if err != nil { log.Println(err) } } } func (env *Env) startBooking(job *Booking, username string, password string, creditCardType string, creditCard string, creditCardCVV string, creditCardExpiry string) bool { timerCtx, cancelTimer := context.WithTimeout(context.Background(), TIMEOUT_MINUTE*time.Minute) defer cancelTimer() headless := os.Getenv("HEADLESS") var u string if strings.ToUpper(headless) == "FALSE" { u = launcher.New(). Set("headless"). Delete("--headless"). MustLaunch() } else { u = launcher.New(). Set("headless"). MustLaunch() } defaultDevice := devices.LaptopWithMDPIScreen // defaultDevice.Screen.Vertical.Height = defaultDevice.Screen.Horizontal.Height // defaultDevice.Screen.Vertical.Width = defaultDevice.Screen.Horizontal.Width defaultDevice.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" browser := rod.New().ControlURL(u).MustConnect().DefaultDevice(defaultDevice) // Close browser when context is done go func() { defer func() { if r := recover(); r != nil { log.Println("Recovering from browser panic...") } }() <-timerCtx.Done() log.Println("Timed out. Closing browser.") browser.MustClose() }() // Defer closing browser defer browser.MustClose() postLoginPage := ktmTrainLogin(browser, username, password) nowTimeStr := time.Now().Format("2006-01-02-15_04_05") postLoginPage.MustWaitLoad().MustScreenshot(fmt.Sprintf("%s/%s-01-login.png", IMAGE_DIR, nowTimeStr)) // Exits if context cancelled select { case <-timerCtx.Done(): browser.MustClose() return false default: } var page *rod.Page getBookingSlotCtx, cancelGetBookingSlot := context.WithTimeout(context.Background(), TIMEOUT_MINUTE*time.Minute) defer cancelGetBookingSlot() pageChan := make(chan *rod.Page) onwardDate := job.TravelDate.Format("2 Jan 2006") timeCode := job.TimeCode name := job.Name gender := job.Gender passport := job.Passport passportExpiry := job.PassportExpiry.Format("2 Jan 2006") contact := job.Contact reverse := job.BackToSg threadCount := 10 for i := 0; i < threadCount; i++ { go func() { defer func() { if r := recover(); r != nil { log.Println("Recovering from getBookingSlot panic...") } }() currPage := getBookingSlots(browser, onwardDate, reverse) log.Println("Booking page loaded.") currPage = selectBookingSlot(getBookingSlotCtx, currPage, timeCode) log.Println("Booking slot selected.") // Make sure page completes loading currPage.MustWaitLoad() select { case <-getBookingSlotCtx.Done(): return default: log.Println("First page loaded.") } // Cancelling context cancelGetBookingSlot() pageChan <- currPage }() time.Sleep(time.Millisecond * 1100) } // Wait for either page received or timeout select { case page = <-pageChan: log.Println("Page received.") case <-timerCtx.Done(): log.Println("Context cancelled.") browser.MustClose() return false } page.MustActivate() // Exits if context cancelled select { case <-timerCtx.Done(): browser.MustClose() return false default: } passthroughLaggingPage := false for !passthroughLaggingPage { filledPassengerDetailContext, filledPassengerDetails := context.WithTimeout(context.Background(), 30*time.Second) go func() { defer func() { if r := recover(); r != nil { log.Println("Recovering from fillPassengerDetails panic...") } }() page = fillPassengerDetails(page, name, gender, passport, passportExpiry, contact) log.Println("Passenger details filled.") filledPassengerDetails() }() <-filledPassengerDetailContext.Done() passthroughLaggingPage = chooseAndMakePayment(timerCtx, creditCardType, creditCard, creditCardCVV, creditCardExpiry, browser, page) // Exits if context cancelled select { case <-timerCtx.Done(): browser.MustClose() return false default: } } // // Start debug screenshots // debugScreenshotCtx, cancelDebugScreenshot := context.WithCancel(context.Background()) // go takeDebugScreenshots(debugScreenshotCtx, courtPage) // // Defer done with debug screenshot // defer cancelDebugScreenshot() time.Sleep(10 * time.Second) _ = page return true } func chooseAndMakePayment(timerCtx context.Context, creditCardType, creditCard, creditCardCVV, creditCardExpiry string, browser *rod.Browser, page *rod.Page) bool { // Exits if context cancelled select { case <-timerCtx.Done(): browser.MustClose() return false default: } cancelRefresh := triggerFuturePageRefresh(page) defer func() { if r := recover(); r != nil { log.Println("Recovering from chooseAndMakePayment panic...") cancelRefresh() } }() if strings.ToUpper(creditCardType) == "VISA" || strings.ToUpper(creditCardType) == "MASTERCARD" { log.Println("Credit card payment method chosen.") page = choosePaymentCreditCard(page) // Wait 5 seconds for payment gateway to load log.Println("Waiting for payment gateway to load.") time.Sleep(time.Second * 5) foundPage := false for !foundPage { for _, currPage := range browser.MustPages() { currPage.MustWaitLoad() var currTitle string err := rod.Try(func() { currTitle = currPage.Timeout(100 * time.Millisecond).MustElement("title").MustText() }) if err != nil { currTitle = "" } if strings.Contains(currTitle, "Payment Acceptance") { log.Println("Page found.") foundPage = true page = currPage } } // Exits if context cancelled select { case <-timerCtx.Done(): browser.MustClose() cancelRefresh() return false default: } time.Sleep(time.Second * 1) } expiryMonth := strings.Split(creditCardExpiry, "/")[0] expiryYear := strings.Split(creditCardExpiry, "/")[1] makePayment(page, creditCardType, creditCard, expiryMonth, expiryYear, creditCardCVV) log.Println("Credit card payment made.") } else if strings.ToUpper(creditCardType) == "KTMWALLET" { log.Println("KTM wallet payment method chosen.") choosePaymentKTMWallet(page) log.Println("KTM wallet payment made.") } cancelRefresh() return true } func ktmTrainLogin(browser *rod.Browser, username string, password string) *rod.Page { page := browser.MustPage("https://online.ktmb.com.my/Account/Login") page.MustElement("#Email").MustInput(username) page.MustElement("#Password").MustInput(password) page.MustElement("#LoginButton").MustClick() return page } func getBookingSlots(browser *rod.Browser, onwardDate string, reverse bool) *rod.Page { page := browser.MustPage("https://shuttleonline.ktmb.com.my/Home/Shuttle") page.MustWaitLoad() // Dismiss system maintenance warning bodyText := page.MustElement("body").MustText() containsCheckStr := "System maintenance scheduled at 23:00 to 00:15" if strings.Contains(bodyText, containsCheckStr) { page.MustEval(`() => document.querySelector("#validationSummaryModal > div > div > div.modal-body > div > div.text-center > button").click()`) } passengerCount := 1 passengerCountStr := strconv.Itoa(passengerCount) requestVerificationToken := page.MustElement("#theForm > input[name=__RequestVerificationToken]").MustAttribute("value") // Get JB Sentral Station Info jBSentralData := page.MustElement("#FromStationData").MustAttribute("value") jBSentralID := page.MustElement("#FromStationId").MustAttribute("value") // Get Woodlands Station Info woodlandsData := page.MustElement("#ToStationData").MustAttribute("value") woodlandsID := page.MustElement("#ToStationId").MustAttribute("value") if !reverse { sensitiveCustomForm(page, *woodlandsData, *jBSentralData, *woodlandsID, *jBSentralID, onwardDate, passengerCountStr, *requestVerificationToken) } else { sensitiveCustomForm(page, *jBSentralData, *woodlandsData, *jBSentralID, *woodlandsID, onwardDate, passengerCountStr, *requestVerificationToken) } page.MustWaitLoad() return page } func selectBookingSlot(ctx context.Context, page *rod.Page, timeCode string) *rod.Page { time.Sleep(5 * time.Second) twoCaptchaAPIKey := os.Getenv("TWOCAPTCHA_API_KEY") needToWait := false // Initial closing of maintenance modal bodyText := page.MustElement("body").MustText() if strings.Contains(bodyText, "System maintenance scheduled at 23:00 to 00:15 (UTC+8)") { closeModalButton := page.MustElement("#popupModalCloseButton") closeModalButton.Eval(`this.click()`) needToWait = true time.Sleep(1000 * time.Millisecond) } // Start probing reportedTicketDetails := false completed := false retries := 2 for !completed { if retries <= 0 { log.Println("Used up retries. Exiting.") completed = true break } page.MustWaitLoad() // Checks for context before proceeding select { case <-ctx.Done(): return page default: } departTripsTable := page.MustElement(".depart-trips") departRows := departTripsTable.MustElements("tr") var rowElement *rod.Element for _, row := range departRows { timeCodeElement := row.MustAttribute("data-hourminute") if *timeCodeElement == timeCode { rowElement = row } } if rowElement == nil { log.Println("No timeslot found. Waiting till context cancelled.") <-ctx.Done() return nil } if !reportedTicketDetails { ticketDetails := rowElement.MustText() log.Printf("Ticket Details: %s", ticketDetails) reportedTicketDetails = true } // Checks for context before clicking select { case <-ctx.Done(): return page default: } // Wait for 1214am if needToWait { log.Println("Waiting for 1214am") destinationTime := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 14, 0, 0, time.Now().Location()) needToWaitTimer := time.NewTimer(time.Until(destinationTime)) select { case <-ctx.Done(): return page case <-needToWaitTimer.C: log.Println("1214am reached") } } selectButtonElement := rowElement.MustElement("a") selectButtonElement.Eval(`this.click()`) page.MustWaitLoad() time.Sleep(500 * time.Millisecond) // Check if there is captcha if strings.Contains(bodyText, "Please complete the reCAPTCHA") { // Reset Body text time.Sleep(500 * time.Millisecond) log.Println("Captcha detected") currURL := "https://shuttleonline.ktmb.com.my/ShuttleTrip" // Regex research for Google Captcha V2 API Key gcaptchaElement := page.MustElement(".g-recaptcha") googleRecaptchaKey := gcaptchaElement.MustAttribute("data-sitekey") log.Printf("Google CAPTCHA V2 Key: %s", *googleRecaptchaKey) captchaAnswer := captchasolver.Provider2CaptchaV2E2E(twoCaptchaAPIKey, currURL, *googleRecaptchaKey) page.MustElement("#g-recaptcha-response").Eval(`this.innerHTML = "` + captchaAnswer + `"`) // Wait for 1215am if needToWait { log.Println("Waiting for 1215am") destinationTime := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 15, 1, 0, time.Now().Location()) needToWaitTimer := time.NewTimer(time.Until(destinationTime)) select { case <-ctx.Done(): return page case <-needToWaitTimer.C: log.Println("1215am reached") } } page.Eval(`RecaptchaCallback()`) } // Check before exiting bodyText := page.MustElement("body").MustText() if strings.Contains(bodyText, "System maintenance scheduled at 23:00 to 00:15 (UTC+8).") { log.Println("Maintenance modal detected on booking page after captcha") completed = false closeModalButton := page.MustElement("#popupModalCloseButton") closeModalButton.Eval(`this.click()`) time.Sleep(1000 * time.Millisecond) retries -= 1 continue } // Repeat if there's no seats -- seats might be released later if strings.Contains(bodyText, "Not enough seat for onward trip") { log.Println("Not enough seat for onward trip") completed = false closeModalButton := page.MustElement("#popupModalCloseButton") closeModalButton.Eval(`this.click()`) time.Sleep(1000 * time.Millisecond) retries -= 1 continue } styleAttribute := selectButtonElement.MustAttribute("style") if styleAttribute == nil || !strings.Contains(*styleAttribute, "display: none") { completed = false } else { log.Println("Completed probing") completed = true } retries -= 1 } // Checks for context before clicking select { case <-ctx.Done(): return page default: } proceedButton := page.MustElement(".proceed-btn") proceedButton.Eval(`this.click()`) return page } func fillPassengerDetails(page *rod.Page, name string, gender string, passport string, passportExpiry string, contact string) *rod.Page { ticketType := "DEWASA/ADULT" nameElement := page.MustElement(".FullName") nameElement.MustInput(name) passportElement := page.MustElement(".PassportNo") passportElement.MustInput(passport) passportExpiryElement := page.MustElement("#Passengers_0__PassportExpiryDate") passportExpiryElement.MustInput(passportExpiry) contactElement := page.MustElement(".ContactNo") contactElement.MustInput(contact) if gender == "M" { maleElement := page.MustElement("#Passengers_0__GenderMale") maleElement.Eval(`this.click()`) } else { femaleElement := page.MustElement("#Passengers_0__GenderFemale") femaleElement.Eval(`this.click()`) } ticketTypeElement := page.MustElement("#Passengers_0__TicketTypeId") ticketTypeElement.MustSelect(ticketType) paymentButton := page.MustElement("#btnConfirmPayment") paymentButton.Eval(`this.click()`) page.MustWaitLoad() return page } func choosePaymentKTMWallet(page *rod.Page) *rod.Page { creditCardButton := page.MustElement("#btnKtmbEWallet") creditCardButton.Eval(`this.click()`) page.MustWaitLoad() return page } func choosePaymentCreditCard(page *rod.Page) *rod.Page { creditCardButton := page.MustElement("#btnGoPaymentPublicBank") creditCardButton.Eval(`this.click()`) page.MustWaitLoad() paymentGatewayButton := page.MustElement("#PaymentGateway") paymentGatewayButton.Eval(`this.click()`) page.MustWaitLoad() return page } func makePayment(page *rod.Page, cardType string, creditCard string, expiryMonth string, expiryYear string, creditCardCVV string) *rod.Page { if cardType == "Visa" { visaRadio := page.MustElement("#card_type_001") visaRadio.Eval(`this.click()`) } else if cardType == "Mastercard" { masterRadio := page.MustElement("#card_type_002") masterRadio.Eval(`this.click()`) } creditCardElement := page.MustElement("#card_number") creditCardElement.MustInput(creditCard) expiryMonthElement := page.MustElement("#card_expiry_month") expiryMonthElement.MustSelect(expiryMonth) expiryYearElement := page.MustElement("#card_expiry_year") expiryYearElement.MustSelect(expiryYear) creditCardCVVElement := page.MustElement("#card_cvn") creditCardCVVElement.MustInput(creditCardCVV) log.Println("Before payment") time.Sleep(10 * time.Millisecond) payButton := page.MustElement(".pay_button") payButton.Eval(`this.click()`) return page } func sensitiveCustomForm( page *rod.Page, fromStationData string, toStationData string, fromStationID string, toStationID string, onwardDate string, passengerCount string, csrf string, ) *rod.Page { defer func() { if r := recover(); r != nil { log.Println("Recovered in sensitiveCustomForm", r) } }() formHTML := fmt.Sprintf(`
UniqueStringHere `, fromStationData, toStationData, fromStationID, toStationID, onwardDate, passengerCount, csrf) page.MustElement("body").MustEval(fmt.Sprintf("() => this.innerHTML = `%s`", formHTML)) // page.MustElement("#presshere").MustClick() page.MustEval(`() => document.querySelector("#presshere").click()`) return page } func triggerFuturePageRefresh(page *rod.Page) func() { ctx, cancelFunc := context.WithCancel(context.Background()) go func() { select { case <-ctx.Done(): return // Reload page after 9 minutes case <-time.After(540 * time.Second): log.Println("Reloading page") page.MustReload() time.Sleep(5 * time.Second) } }() return cancelFunc }