diff --git a/.bin/wordlist-updaters/status.json b/.bin/wordlist-updaters/status.json
index d59fd7de..49b321f7 100644
--- a/.bin/wordlist-updaters/status.json
+++ b/.bin/wordlist-updaters/status.json
@@ -3,6 +3,6 @@
         "last_update": 1732302193
     },
     "Trickest wordlist update": {
-        "last_update": 1732442719
+        "last_update": 1732529050
     }
 }
\ No newline at end of file
diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt
index 8033ee5a..2894ebba 100644
--- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt
+++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt
@@ -2775,8 +2775,8 @@ hotelfinder/rpc
 hotels/rpc
 hp/video
 hpc/*/supported-applications.aspx?
-http://amzn.to/robots.txt
-http://bit.ly/robots.txt
+http://wp.me/robots.txt
+http://yelp.com/robots.txt
 hub?category=
 hz/contact-us/ajax/initiate-trusted-contact
 hz/help/contact/*/message/$
@@ -4226,7 +4226,7 @@ shopping/seller
 shopping/suppliers/search
 shopping?
 shopstream
-showtimes/location/*http://instagram.com/robots.txt
+showtimes/location/*http://apis.google.com/robots.txt
 sidewiki/entry
 signature
 signin/inject
diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt
index 4f326821..a02842c7 100644
--- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt
+++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt
@@ -3121,7 +3121,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /newssearch
 /no-internet-heading-assigned
 /no-internet-heading-assisted
-/no-referrer-when-downgradehttp://gmail.com/robots.txt
+/no-referrer-when-downgradehttp://steamcommunity.com/robots.txt
 /no-search-results
 /node
 /node/add
@@ -3713,7 +3713,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*__rmid
 ?*__rmidpage
 ?*ajs_event
-?*apikey*http://visitazores.com/robots.txt
+?*apikey*http://boredpanda.com/robots.txt
 ?*app
 ?*appidt=*
 ?*aspects=
@@ -3830,7 +3830,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*utm_medium
 ?*utm_source
 ?*utm_source=
-?*utm_source=http://chrome.google.com/robots.txt
+?*utm_source=http://pixel.quantserve.com/robots.txt
 ?*verified=
 ?*xid
 ?252525253Bpreview=
@@ -4330,8 +4330,10 @@ AcceptDeferredTermsAndConditions
 AccommodationCrossSells
 AccommodationTips
 Account
+AccountAccess
 AccountDisavow?
 AccountMerge
+AccountStatus
 Achievements
 ActLikeAnOwner
 ActionRecord
@@ -4413,6 +4415,7 @@ App_Data/
 App_Themes/
 App_config/
 AppleMapsAction
+Arabic
 Archive/New_Records_Testing
 Archive/PrintMaterial/ACS_Journal/1/files
 Archive/PrintMaterial/ACS_Journal/10/files
@@ -5271,6 +5274,7 @@ BeachSweeps
 Bender
 Benoit/Public/DVDDB
 BetterBySea
+BillingActivity
 BillingAndPayments
 BillingDetail
 BillingInvoiceFile
@@ -5297,12 +5301,14 @@ CampaignConfiguration
 CampaignId
 CampaignManagement
 CancelBooking
+CancelPlan
 CaptchaValidationLog
 CarsComparePartners
 Cart
 CauseTeaserPopup
 Celebrate100Million
 CertificateOfExcellenceRequestSupport
+ChangePlan
 CheapFlightsCommerce
 CheapFlightsController
 CheapFlightsHome
@@ -5317,6 +5323,7 @@ CheckoutEditItemsDisplayView
 CheckoutOrderBillingView
 CheckoutOrderShippingView
 CheckoutSignInView
+Chinese
 Citation/Download
 Civ
 Claim
@@ -5389,6 +5396,7 @@ Disallow
 Disallow:
 DiscountGroup
 DisplayMap
+DoNotTest
 Doc
 Domains/DomainControlPanel
 Download
@@ -5399,12 +5407,14 @@ ESPDisplayOptionsViewCmd
 ESPModal
 EULegal
 EarlyRequesterJS/static-1.38
+EditProfiles
 ElasticLoadBalancing/2009-05-15
 ElasticLoadBalancing/2010-07-01
 EmailCartView
 EmailHotel
 EmailMap
 EmailMemberRegistration
+EmailPreferences
 EmailSpellCheckJson
 Emailer
 EmbedSlideshow
@@ -5438,6 +5448,7 @@ FatalError.html
 FavoriteReviewManagement
 Featured
 FeaturedShowMoreOverlay
+Features/SpinaBifidaProgram/
 Feeds
 Fetch
 FetchProdRefreshContent
@@ -5445,6 +5456,7 @@ FiatsCmd
 FilterState
 FindRestaurants
 FinishAppInstall
+FireTV
 Flamingo_SearchEngine
 Flights?
 FlightsCommerce
@@ -5515,6 +5527,7 @@ HRMConnect
 HTTrack
 HYx10rg3
 HandleHotelSuperPortal
+HdToggle
 HeadJS/static-2.115
 HeadJS/static-2.83
 HeadJS/static-2.84
@@ -5540,6 +5553,7 @@ HotelsManageListing
 HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
+HpImageArchive.aspx
 HubStyle/static-2.309
 HubStyle/static-2.313
 HubStyle/static-2.315
@@ -5639,6 +5653,7 @@ JustForYouGuide
 Jyxobot
 Kimengi
 Kimengi/nineconnections.com
+Korean
 LFE/Live.controls.sharedpages/error.aspx
 LICENSE.txt
 LNAjax
@@ -5646,6 +5661,7 @@ LP
 LanderLogger
 LandingPage
 Landingpages.html
+LanguagePreferences
 Leaderboard
 LexxeBot/1.0
 Library
@@ -5667,7 +5683,9 @@ LocationTab
 LocationWidget
 LogMediaClick
 Login.html
+LoginHelp
 LogonForm
+Logout
 MAINTAINERS.txt
 MAPS
 MJ12bot
@@ -5679,10 +5697,12 @@ MachineTranslation
 Mail/
 MailEmergency
 Manage/
+ManageDevices
 ManageListing
 ManageOrder
 ManageOwnersTopReasonsApi
 ManagePhotos
+ManageProfiles
 ManageReturns
 ManageShopListingApi
 ManageShopping
@@ -5694,6 +5714,7 @@ MapS
 Maps
 MarketPerception
 Marketing/EcapSurvey.aspx
+Maxis
 Maxthon
 McKinsey
 MediaBadgeResponse
@@ -5771,6 +5792,7 @@ MemberSubscribe
 MemberSubscriptionsController
 MemberTagsAjax
 MemberTravelMap
+Meo
 MessageRunDetailID=
 Messages
 MessagingOptions
@@ -5916,6 +5938,7 @@ ModuleAjax
 ModuleBootstrap
 Moreover
 Moreoverbot
+MoviesYouveSeen
 MptUrl
 MsnForumFeedServlet
 MultiOwners
@@ -5951,6 +5974,7 @@ NetSeer
 NewMessage
 NewOwners
 NewTopic
+NewWatchInstantlyRSS
 News
 News/Flash.aspx/
 News/Index
@@ -6022,6 +6046,7 @@ PacMan
 PageBuilderControls
 PageSpeed=noscript
 PageviewRecord
+Panasonic
 PanoramaXml
 Partners
 PasswordReset
@@ -6055,6 +6080,7 @@ PoiRecommendationEditor
 PoiRecommendationList
 PoiReviewDetail
 PoiTipDashboard
+PopupDetails
 PortalIdParser/static-1.27
 PortalIdParser/static-1.28
 PortalIdParser/static-1.29
@@ -6094,10 +6120,12 @@ Products/Reports.aspx
 Products/ShoppingCart.aspx
 Products/subscriptionsSpecial.aspx
 Profile
+ProfilesGate
 Project
 PromoTools
 PromotionDetailsDisplayView
 PromotionDisplayView
+Proxy.ashx
 ProxyToolsAccountSettingsTool
 ProxyToolsBillingAndPaymentsTool
 ProxyToolsBillingDetailTool
@@ -6231,6 +6259,7 @@ Saves
 ScreensaverShade
 Scripts
 Search
+Search
 Search/*
 Search/Mail/Public
 SearchNavigationView
@@ -6293,6 +6322,7 @@ Snap
 SocialIapReporting
 Societies
 SolutionsDisplay
+Sony
 SortAttractions
 SortEateries
 SortHotels
@@ -6377,6 +6407,15 @@ Team
 TeamPresent
 Teleport
 TeleportPro
+TemplatePackage/3.0/Templates
+TemplatePackage/3.0/examples
+TemplatePackage/4.0/docs
+TemplatePackage/4.0/gallery-internal
+TemplatePackage/Templates
+TemplatePackage/examples
+TemplatePackage/gadgets
+TemplatePackage/modules
+TemplatePackage/subtopic
 TemporarilyClosedNoticeApi
 Test
 TestEmptyPage
@@ -6523,6 +6562,7 @@ WebReaper
 WebStripper
 WebZIP
 WhatsNearby
+WiViewingActivity
 WidgetBrowserExtension
 WidgetEmbed
 WidgetListAsync.naver
@@ -6540,6 +6580,8 @@ Xenu
 Xhr
 Yandex
 Yasni
+YourAccount
+YourAccountPayment
 Zao
 Zealbot
 ZendeskRedirector
@@ -6621,8 +6663,10 @@ _api/dealer-offer-events-service/proxy/v1/dealer-offer-events
 _assets/img/favicons
 _blank
 _blog_rss
+_borders
 _comments
 _csrf
+_derived
 _email.html
 _escaped_fragment_
 _escaped_fragment_=aboutgroup
@@ -6633,6 +6677,7 @@ _escaped_fragment_=search
 _escaped_fragment_=searchin
 _escaped_fragment_=starred
 _finance_doubledown
+_fpclass
 _frdtcr
 _ga
 _ga=
@@ -6681,11 +6726,13 @@ _nimbusJsonp.js
 _nimbusJsonpStatic.js
 _nitf.xml
 _ok
+_overlay
 _partial
 _partials/wix-vod
 _partners
 _print
 _print.html
+_private
 _private/admin-tools
 _prq
 _ptid=
@@ -6713,9 +6760,16 @@ _td_api
 _tdhl_api
 _tdpp_api
 _template
+_themes
 _uac/adpage.html
 _v1st
 _vercel/insights/view
+_vti_bin
+_vti_cnf
+_vti_log
+_vti_map
+_vti_pvt
+_vti_txt
 _zc
 a
 a/*.*/about
@@ -6821,6 +6875,8 @@ ac/
 ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search
 ac2
 academic-subscriptions
+academic/profile
+academic/search
 academy
 academy-free-account
 academy/*/results
@@ -6884,6 +6940,7 @@ account_settings/set_password?
 account_settings/set_your_password$
 account_settings/set_your_password?
 account_settings?
+accountaccess
 accounthelp
 accounts
 accounts/ClientAuth
@@ -6896,6 +6953,7 @@ accounts/login
 accounts/new
 accounts/o8
 accounts/o8/id
+accountstatus
 acct
 ach/forums
 ach/kb/*/discuss
@@ -6903,6 +6961,7 @@ ach/search
 ach/user
 ach/users
 achievements
+aclick
 aclk
 acoustic
 acq/nonprofits
@@ -7113,6 +7172,7 @@ ai
 ai_log
 aihub/tfhub
 aim
+airtel
 ajah
 ajax
 ajax/*
@@ -7187,6 +7247,7 @@ alexa-top-sites/*
 alexa-web-search/*
 alexa/*
 alexaforbusiness/
+alink
 alkali
 all
 all-supported-params
@@ -7213,6 +7274,7 @@ amp
 amp$
 amp-proxy
 amp-tealium
+amp/
 amp_next_page_ga.cms*
 amphtml
 amphtml/blogs/test
@@ -7244,6 +7306,7 @@ analytics/portal
 analytics/seomagic
 analytics/traffic/overview
 analytics/uploads
+anan
 and
 android
 android/downloads
@@ -7284,13 +7347,14 @@ api/editionurlgenerator
 api/eventlogging
 api/federation
 api/html
-api/http://kobo.com/robots.txt
+api/http://nam.edu/robots.txt
 api/infinite-list.html
 api/internal/*/location
 api/internal/_bb
 api/internal/analytics/songs
 api/jobPostings/jobs
 api/legendary-data/relay
+api/maps/
 api/nextsteps
 api/nicorepo/timeline
 api/partner_select/*/price_query
@@ -7365,7 +7429,7 @@ appchart
 appdynamics-wem.cisco.com
 appealed-pin
 apple-app-site-association
-apple-app-site-associationhttp://fonts.google.com/robots.txt
+apple-app-site-associationhttp://mediafire.com/robots.txt
 apple-news
 appliance
 applicant-eligibility
@@ -7781,6 +7845,7 @@ audio_ad?
 audio_file
 audio_news
 audio_player
+aui/inbound
 aura
 aura/promo
 auth
@@ -7790,6 +7855,7 @@ auth/*
 auth/createchallenge
 auth/sso/proxy-login
 auth/verifychallenge
+authenticate
 authentication
 authentication$
 authkey=
@@ -7809,6 +7875,7 @@ autocomplete/projects
 autocomplete/users
 autocompletes
 autodiscover/
+autologin
 autoplay=
 autor/francisco_jose_pedrosa_rodriguez/a/*/
 autor/francisco_jose_pedrosa_rodriguez/a/ 
@@ -8170,7 +8237,6 @@ b?*node=7454939011
 b?*node=9052533011
 ba
 baby-and-child/archive/
-backend
 backoffice=
 backstage/api/
 badges/profile/create
@@ -8424,6 +8490,7 @@ bezahlvorgaenge.html
 bfe/apps/website
 bfl
 bfmp
+bfp/search
 bg
 bg-*/
 bg-bg/
@@ -8454,6 +8521,7 @@ bin
 bin/
 bin/search.go
 bin/search.py
+bing-site-safety
 bingbot
 bipolar-disorder/bipolar-depression-management
 bipolar-disorder/video/bipolar-fatherhood
@@ -8543,7 +8611,7 @@ blog/*?text=
 blog/*sitemap.xml$
 blog/*utm_source=
 blog/.
-blog/.html.*http://salon.com/robots.txt
+blog/.html.*http://lds.org/robots.txt
 blog/?p=
 blog/?s=
 blog/Neuer_Beitrag.html
@@ -8583,6 +8651,7 @@ blogs/blogpost/preview
 blogs/nationals-journal-beta
 blogs/pad$
 blogs/pad/$
+blogs/search/
 blogs/slow-ride
 blogs/technology.atom
 blogs/test
@@ -8650,7 +8719,7 @@ bookmark_mana2
 bookmarklet
 bookmarks
 bookmarks/widget
-bookmarkshttp://ed.ted.com/robots.txt
+bookmarkshttp://spiegel.de/robots.txt
 books
 books/NBK
 books/about
@@ -8673,6 +8742,7 @@ books?*zoom=5
 bookshelf
 booksrightsholders
 boutique/sitemaps
+bouyguestelecom
 bp
 bp/c
 bp/guest_mfg_brand
@@ -8748,9 +8818,11 @@ brc/
 breakingnews/toihome.html
 breast-cancer/bc-treatment-21/provider-select
 briefEditorPublicationId=
+britishairways
 browse
 browse-2
 browse.aspx
+browse/genre
 browse/homes/all
 browse/more_page
 browse/reviews/recent
@@ -8762,6 +8834,7 @@ browse_content
 browser/tutorials
 browsers
 bruce
+brutus
 brw
 bs/android/collections
 bs/android/search
@@ -9007,7 +9080,6 @@ c360/v1/track
 c360_v1_track
 c?*attr_*=
 cDN_contENt
-cHash
 ca
 ca*/blog-article-remove-faq-utms-*.js
 ca*/cdn-cgi/challenge-platform
@@ -9563,7 +9635,7 @@ cdn-cgi
 cdn-cgi/
 cdn-cgi/bm/cv
 cdn-cgi/challenge-platform
-cdn-cgi/http://mediapost.com/robots.txt
+cdn-cgi/http://fda.gov/robots.txt
 cdn-cgi/rum
 cdn-cgi/zaraz/s.js
 cdn-cgi/zaraz/t
@@ -9591,7 +9663,7 @@ cgi-bin/loginpage
 cgi-bin/search.fpl
 cgi-bin/state
 cgi-bin/wx
-cgi-binhttp://ifttt.com/robots.txt
+cgi-binhttp://worldbank.org/robots.txt
 cgi-local
 cgi-perl
 cgi-registry
@@ -9657,6 +9729,7 @@ change-email
 change-password
 change-shipping
 changebrowser
+changeplan
 channels
 chappai
 chapter
@@ -9728,6 +9801,7 @@ checkpoint
 chemo-combo
 chiaki1007akie/videolist-*.html
 chiaki1007akie/videolist.html
+chinese
 chisla.html
 chistory.php
 chocolatechip
@@ -9785,6 +9859,7 @@ civis/posts
 civis/search
 civis/threads/*in_iframe=1
 civis/threads/*wp_data=
+ck/
 cl
 cl2/feeds
 cl2/ical
@@ -9810,6 +9885,7 @@ classnames/static-2.3
 classroom
 clck
 clean
+clearcookies
 cli
 click
 clickId=
@@ -9934,7 +10010,7 @@ co/users
 co/ventureone
 code
 code/*
-code/http://w3techs.com/robots.txt
+code/http://bloomberg.com/robots.txt
 codeCoverage
 codesamples/
 coin*mid=
@@ -9980,6 +10056,7 @@ comeuntochrist/*/internal-use-only/
 comeuntochrist/*/search-results/
 comeuntochrist/lp/
 comeuntochrist/teach/
+comhem
 coming-soon*language=
 comm
 comm-part-test
@@ -10058,6 +10135,7 @@ community_comment
 community_post
 community_s
 compagnie
+companies
 companies/c
 companies/o
 companies/search
@@ -10126,6 +10204,7 @@ computer/thanks-partner
 comscore-json
 comunes
 concursocampusparty/files
+concussion/HeadsUp/training/
 conditions/coronavirus-covid-19/testing-and-tracing/get-an-antigen-test-to-check-if-you-have-coronavirus/
 conditions/pregnancy-and-baby/pages/your-birth-plan.aspx
 conf
@@ -10147,7 +10226,7 @@ confirmation.html
 conflagexp
 connect
 connect/@
-connect/http://blog.twitter.com/robots.txt
+connect/http://venturebeat.com/robots.txt
 connect/social-intranet.html
 connections
 connections/recent
@@ -10264,7 +10343,7 @@ content/dam/ircc/documents/pdf/francais/trousses/form/imm5476
 content/dam/ircc/documents/pdf/francais/trousses/form/imm5644
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0002
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0004
-content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://canalplus.fr/robots.txt
+content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://nydailynews.com/robots.txt
 content/dam/ircc/documents/pdf/francais/trousses/guides/guide-0142-avion-afghanistan-voie-acces-rp.pdf
 content/dam/oecd/
 content/dam/redcross/lp
@@ -10420,6 +10499,7 @@ coupons.
 coupons/visit
 courier
 courses/
+courts
 cover
 covid19
 covid19/mobility
@@ -10439,7 +10519,9 @@ cpresources
 cpro
 cpropub/univercd
 cpspa
+cr$
 cr/
+cr?
 craft
 crash
 crawler4j
@@ -10518,6 +10600,7 @@ cse?
 cservice
 cservice/ya_index.gsp
 csi_204
+csl
 csmlists/addbookmark
 csmlists/topic
 csp
@@ -10947,6 +11030,7 @@ device
 device-ajax
 device-prefs
 deviceList
+deviceManagement
 devmailmobile/
 devmx/
 devopsconsole/images
@@ -11071,7 +11155,6 @@ display-video-360
 dispute
 disputenotice
 dist
-distributions
 dk
 dk$
 dk*/blog-article-remove-faq-utms-*.js
@@ -11436,6 +11519,7 @@ editor.jsp
 editor2.jsp
 editorial-images
 editorial/*
+editpayment
 edu/cs4hs
 edu/ping
 edu/tasks
@@ -11486,6 +11570,7 @@ element/commun/afficher
 eli-search
 eliminar-busqueda
 eliminar-favoritos
+elisa
 elite?
 elp
 eltiempo/buscador
@@ -11499,7 +11584,7 @@ email-preferences/
 email-unsubscribe
 email-validation
 email/$
-email/*http://dw.com/robots.txt
+email/*http://timesofindia.indiatimes.com/robots.txt
 email/EnrollmentUnsubscribe.htm
 email/SubscriptionTypeUnsubscribe.htm
 email/Unsubscribe.htm
@@ -11521,6 +11606,7 @@ emailpreferences.aspx
 emails
 emailservice
 emailsignup
+emailunsubscribe
 embed
 embed/
 embed/*
@@ -11718,13 +11804,18 @@ enterprise/blog/search$
 enterprise/blog/search?
 enterprise/orgs-terms
 enterprise/seo
+entertain
 entertainment/article/Minerva-s-horoscope
 entertainment/bravo
+entities/search
+entityexplore$
+entityexplore?
 entrez
 entrez/eutils
 entrez/sutils
 entrez/utils
 entrez/viewer.fcgi
+entrytrap
 enus/
 enviro/static-3.70
 enviro/static-3.72
@@ -12086,6 +12177,7 @@ eu/promos
 eu/search
 eu/user
 eu/users
+eula
 eur-lex
 eures/main.jsp
 europeaid/companion
@@ -12244,6 +12336,7 @@ fbml/ajax/dialog
 fbpostaction.cms?article=
 fcgi-
 fckeditor
+fd/
 fdbk
 feature/sf-test
 feature_nine_browse-bin=
@@ -12426,7 +12519,7 @@ files/techabuse-welsh.pdf
 files/techabuse.pdf
 fileview
 filing/free-file-fillable-forms/free-file-fillable-forms-support
-filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://is.gd/robots.txt
+filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://globalresearch.ca/robots.txt
 filter/tips
 filter/tips
 filterTags=
@@ -12454,6 +12547,7 @@ firefox
 firefox/?
 firefox/downloads
 firefox/new_tab_learn_more
+firetv
 firststeps.
 fisheries/fleet
 fix-it
@@ -12477,6 +12571,7 @@ flickrimagerequest
 flights
 flights/index.hi.
 flipboardrss
+flu/espanol/
 flutter-canvaskit
 fn_adspaces
 fol
@@ -12785,6 +12880,7 @@ free-video/*/square
 free-video/*/vertical
 free/cloud-computing-free-tier/
 free/databases-free-tier/
+freemonth
 freertos/archive
 french-studies/
 fresh
@@ -12805,6 +12901,8 @@ full
 fullscreen
 fullscreen/ReportLinker
 fun
+fun/api/
+fun/g/
 fundraiser/charity
 fundraiser/search
 fur/forums
@@ -12826,7 +12924,7 @@ fy-NL/search
 fy-NL/user
 fy-NL/users
 g
-g/4arE9Chttp://flickr.com/robots.txt
+g/4arE9Chttp://nngroup.com/robots.txt
 g/api
 g00
 g01
@@ -13019,6 +13117,7 @@ geo/tools
 geocoder
 geoip.php
 geolocation
+geooverride
 get
 get$
 get-author-review
@@ -13382,7 +13481,7 @@ ha/search
 ha/user
 ha/users
 hackathons/reinvent-2013/
-hacker/*/followhttp://hackerone.com/robots.txt
+hacker/*/followhttp://s3-eu-west-1.amazonaws.com/robots.txt
 hacker/*/unfollow
 hair-transplant-services
 handler
@@ -13715,12 +13814,14 @@ hosted
 hosted/images
 hosting/search
 hotel/index.hi.
+hotel/reviews?
 hotel/us/the-airstream-van.*.html
 hotel_attractions
 hotel_rt_onview
 hotelfinder/rpc
 hotels
 hotels/rpc
+hotels/search
 hotelsonmap.*.json
 hotfix/
 houseads
@@ -13735,6 +13836,8 @@ howto/GoodNeighborPolicy.htm
 howtobuy/
 hp/focus-retrocompatibile$
 hp/video
+hpm
+hpmob
 hr
 hr-*/
 hr-hr/
@@ -13784,43 +13887,43 @@ http
 http-checker
 http2_data
 http:
-http://2.bp.blogspot.com/robots.txt
-http://about.google/robots.txt
-http://amazon.com/robots.txt
-http://amazon.de/robots.txt
-http://amazon.in/robots.txt
-http://ameblo.jp/robots.txt
-http://amzn.to/robots.txt
-http://api.whatsapp.com/robots.txt
-http://app.box.com/robots.txt
-http://bbc.com/robots.txt
-http://bcn.cl/robots.txt
-http://beatport.com/robots.txt
+http://1.gravatar.com/robots.txt
+http://answers.com/robots.txt
+http://apis.google.com/robots.txt
 http://behance.net/robots.txt
-http://census.gov/robots.txt
-http://creativecommons.org/robots.txt
-http://euronews.com/robots.txt
-http://gist.github.com/robots.txt
-http://graphisoft.com/robots.txt
-http://he.wikipedia.org/robots.txt
-http://hubspot.com/robots.txt
-http://ie.edu/robots.txt
-http://mrisoftware.com/robots.txt
-http://nationalgeographic.com/robots.txt
-http://news.yahoo.com/robots.txt
-http://patreon.com/robots.txt
-http://producthunt.com/robots.txt
-http://raiplay.it/robots.txt
-http://redbubble.com/robots.txt
-http://soundcloud.com/robots.txt
-http://support.apple.com/robots.txt
-http://techradar.com/robots.txt
-http://theverge.com/robots.txt
-http://tiny.cc/robots.txt
-http://torproject.org/robots.txt
-http://translate.google.com/robots.txt
-http://vogue.com/robots.txt
-http://yelp.com/robots.txt
+http://developer.apple.com/robots.txt
+http://developer.yahoo.com/robots.txt
+http://eepurl.com/robots.txt
+http://eff.org/robots.txt
+http://europarl.europa.eu/robots.txt
+http://foxnews.com/robots.txt
+http://gartner.com/robots.txt
+http://getbootstrap.com/robots.txt
+http://google.co.in/robots.txt
+http://google.fr/robots.txt
+http://hackaday.io/robots.txt
+http://hootsuite.com/robots.txt
+http://i2.wp.com/robots.txt
+http://ikea.com/robots.txt
+http://img.youtube.com/robots.txt
+http://inc.com/robots.txt
+http://jstor.org/robots.txt
+http://justgiving.com/robots.txt
+http://lastpass.com/robots.txt
+http://latimes.com/robots.txt
+http://lifehacker.com/robots.txt
+http://news.google.com/robots.txt
+http://penguinrandomhouse.com/robots.txt
+http://roguefitness.com/robots.txt
+http://salon.com/robots.txt
+http://sfchronicle.com/robots.txt
+http://skype.com/robots.txt
+http://upload.wikimedia.org/robots.txt
+http://us02web.zoom.us/robots.txt
+http://usnews.com/robots.txt
+http://yandex.ru/robots.txt
+http://yumpu.com/robots.txt
+http://zendesk.com/robots.txt
 hu
 hu-*/
 hu-hu/
@@ -14158,6 +14261,7 @@ image-*tool=
 image-server
 image/detailprint
 imageClipperUpload
+imagelibrary
 images
 images-apphost
 images-data
@@ -14167,11 +14271,16 @@ images/$
 images/*
 images/?
 images/Podcasts
+images/async/?
+images/async?
 images/carBlogsimages
 images/chart_gifs
 images/chart_pdfs
 images/chart_png300
 images/chart_png460
+images/hosted/?
+images/hosted/search?
+images/hosted?
 images/interview_pdfs
 images/li.gif
 images/logging_requests.gif
@@ -14180,8 +14289,13 @@ images/ndtvvideo/*
 images/pad/search?rpt=imageview
 images/pad/search?text=*&source=morda
 images/resume
+images/search%3F
+images/search%5C
+images/search/?
+images/search?
 images/search?rpt=imageview
 images/search?text=*&source=morda
+images/searchbyimage
 images/showroomimages
 images/smart/$
 images/touch/$
@@ -14192,6 +14306,9 @@ images/usedcarimages
 images/userprofileimg
 images/v$
 images/viewconfig$
+images?
+images?*mmbaseline*
+images?*nr*
 imagine/
 imesync
 imf-fetch-message
@@ -14318,6 +14435,7 @@ inapp
 inbox
 inbox$
 inbox$
+inboxcommerce
 inc5000dev
 include
 include/
@@ -14454,7 +14572,7 @@ integrate/
 integrations
 integromedb
 intelligencer/amp
-interactive*http://fda.gov/robots.txt
+interactive*http://artstation.com/robots.txt
 interactive/reading-preference
 interactives
 interface/pda
@@ -14671,7 +14789,7 @@ it/search*price_bucket=
 it/search*search_type=
 it/search*ship_to=
 it/search/autocomplete
-it/searchhttp://m.facebook.com/robots.txt
+it/searchhttp://www1.nyc.gov/robots.txt
 it/share
 it/shop_name_search_service
 it/solutions/case-studies/
@@ -15193,6 +15311,8 @@ ko/user
 ko/users
 ko_kr/templates/M
 kolkata
+korean
+kpn
 kr*/blog-article-remove-faq-utms-*.js
 kr*/cdn-cgi/challenge-platform
 kr*/step
@@ -15290,7 +15410,7 @@ layout/set/txt_pq
 layout/set/xml
 layouts
 lb
-leadingtogether/http://validator.w3.org/robots.txt
+leadingtogether/http://hayhouse.com/robots.txt
 leaf
 learn
 learn-noperf
@@ -15324,6 +15444,8 @@ learning/settings
 learning/settings$
 learning/settings?
 learnmore
+learnmorelayer
+learnmorelayertv
 lecture
 lees/gids/exclusief
 lees/gids/exclusief$
@@ -15514,6 +15636,7 @@ loans
 loc/volunteers
 loc=
 local
+local
 local-services-ads
 local/ajax
 local/blogsandcolumns/slow-ride-front
@@ -15590,7 +15713,10 @@ login_or_join/redirected$
 login_or_join/redirected?
 login_or_join?
 login_with_redirect
+loginhelp
 loginhelp/
+loginhelp/lookup
+loginhelpsucess
 loginidentification/
 logo
 logo-maker/brief
@@ -15777,8 +15903,6 @@ manageyourkindle
 manifest.json
 manifest.webmanifest
 manotest/
-manual/add-note.php
-manual/vote-note.php
 map
 map-constructor/loader
 map-regular.xml
@@ -15800,6 +15924,7 @@ maps/*?lang=uk$
 maps/*?lang=uz$
 maps/*?ll=
 maps/about
+maps/adsendpoint
 maps/api/js
 maps/api/place/js
 maps/api/staticmap
@@ -15862,11 +15987,13 @@ mats/?diagid=100001&eulaaccept=yes
 mats/default.aspx?diagid=100001&eulaaccept=yes
 max=
 max_price=
+maxis
 maxprice=
 mb/
 mbf
 mbox
 mbr/
+mcd
 me
 mecha
 media
@@ -15926,7 +16053,7 @@ media/tourism-content
 media/travelpod-roll.flv
 media/tv/channel/credits
 media/tyfo*
-media/tyfo*http://statcounter.com/robots.txt
+media/tyfo*http://codex.wordpress.org/robots.txt
 media/unesco
 media/vr-geo-photo-j
 media/vr-geo-photo-l
@@ -16242,6 +16369,7 @@ memorandum
 mental-health-genetic-testing
 menu
 menu/*/feedback_form
+meo
 meps/*/eptvAjax
 meps/*/pdf*
 meps/*/xml*
@@ -16253,6 +16381,7 @@ merchant/directions
 merchant/imageUploadPreview
 merchant/loc
 merchant/moreProducts?
+merchant/reviews?
 merchant/update
 merchantapps
 mes
@@ -16288,6 +16417,7 @@ microsoft/bizspark/
 microsoft/bizspark/signon/thankyou/
 middle
 migration
+migration/
 mikly.pl
 mimeType=
 min=
@@ -16375,7 +16505,6 @@ mobilepagenotfounderror.aspx
 mobileservererror.aspx
 mockup-editor
 mod
-mod.php
 mod=
 mod?
 modal-message
@@ -16391,6 +16520,7 @@ modern-formats-eng/ucp/static/*/client
 modern-formats-eng/ucp/static/*/csr
 modern-formats-eng/ucp/static/*/dev
 modern-formats-eng/ucp/static/framework_logging
+modernizr
 modules
 modules/*.css$
 modules/*.css?
@@ -16964,6 +17094,7 @@ ngc=
 nii=t
 nike
 nimbus_ms/remote?
+niosh/archive/
 nl
 nl*/blog-article-remove-faq-utms-*.js
 nl*/cdn-cgi/challenge-platform
@@ -17253,12 +17384,16 @@ noscript
 not-allowed
 not-supported.html
 not_recommended_reviews
+notamember
+notfound
 notfound.html
+notices
 noticias/20080416/destituyen-director-jefe-proteccion-asco-tras-fuga-radiactiva/28170.shtml
 notificarelacionadas
 notifications
 notifications
 notifications$
+notifications/
 notifications/*
 nova
 now/mt3
@@ -17268,6 +17403,7 @@ npk
 npln
 npost/
 nqrwns
+nrdapp
 ns/
 ns/apology
 ns/hydra/console/?
@@ -17447,6 +17583,8 @@ offers/nonprofits
 offers/nonprofits$
 offers/nonprofits%
 offers/nonprofits?
+offers/proxy/dealsserver/api/log
+offers/proxy/dealsserver/buy
 offers/referral-terms-conditions
 offers/referral-terms-conditions$
 offers/referral-terms-conditions%
@@ -17499,6 +17637,7 @@ or/search
 or/user
 or/users
 oral-and-maxillofacial-surgeon
+orangefrance
 order
 order-detail
 order-results
@@ -17537,6 +17676,7 @@ over-de-vpro/bedankt.*$
 over-de-vpro/bedankt.html$
 overlay
 overview
+overviewblockseeother
 owner_settings
 p
 p.php
@@ -17907,6 +18047,7 @@ paid-posts
 palinsesto/guidatv/*/*.html$
 palinsesto/guidatv/lista/*/*.html$
 palinsesto/retrocompatibile/*/*.html
+panasonic
 panel
 panelweb
 panorama/welt/playboy_aid_119773.html
@@ -18092,6 +18233,7 @@ phbr1-
 phbr2-
 phbr~r_
 phone/compare/?
+phonenumber
 photo
 photo-editor
 photo-test-editor-x
@@ -18133,6 +18275,7 @@ pictures/related
 pictures/search
 pift/retest
 pigeon
+pin
 pin-builder
 pin-communities
 pin-creation-tool
@@ -18331,6 +18474,7 @@ player/metadata/video
 playerColor=
 playlist.php
 playlists/*/user
+playstation
 plenary/*/vod.html
 plonejsi18n$
 pls
@@ -18625,8 +18769,11 @@ polling
 polopolydevelopment
 pools/c
 pools/campaign
+popup/codewhatisthis
 popup?
 popup_security.jsp
+popupdetails
+popupprivacypolicy
 popups/mediaplayer
 popups/popup_gallery
 portal
@@ -18758,9 +18905,10 @@ privacy/consent
 privacy/request-data
 privacy/zoho-group.html
 privacy_policy
+privacypolicychanges
 private
-private/http://sfgate.com/robots.txt
-private/http://shareasale.com/robots.txt
+private/http://csmonitor.com/robots.txt
+private/http://wunderground.com/robots.txt
 private/wtrack
 prix-essence
 pro
@@ -18770,6 +18918,7 @@ pro-resources
 pro-site
 pro/about
 pro2
+proFile/history?
 proMatch
 proProjects
 proVideoWidget
@@ -18787,6 +18936,7 @@ product-reviews/B0069IY63Y
 product/electrode/api/logger
 product/electrode/api/wpa
 product/recommended
+product/reviews?
 productBrand=
 productId=
 productPurchaseDone
@@ -18838,6 +18988,7 @@ profile/*/following
 profile/*/map$
 profile/*/photos
 profile/downloads_limit
+profile/history?
 profile/license
 profile/payment/getstarted
 profile/view
@@ -18919,6 +19070,7 @@ protected
 protocol
 protocolexchange/labgroups
 proximic
+proximus
 proxy
 proxy
 prp
@@ -19188,6 +19340,7 @@ q/user/*/subscribers
 q/user/*/verifies
 q=
 qa
+qantas
 qemail
 qi
 qnasearch?
@@ -19394,6 +19547,7 @@ relations.php
 releasenotes/item/
 relevance_feedback
 relnotes
+rememberme
 remote-login.php
 remote.control
 remotecontrol/bestpractices/jl-prototype
@@ -19485,6 +19639,7 @@ resources/xml/sitemaps
 responder
 rest
 results
+results
 results-export-email-by-search-data
 results-export-ids
 results-export-search-by-year
@@ -19541,6 +19696,7 @@ reviews/write/?s=
 reviews/write?s=
 reviews_page=
 rewards
+rewardsapp/
 rf=*
 ribbon.php
 rip/
@@ -19750,6 +19906,7 @@ salesImages/static-1.382
 salesImages/static-1.383
 salesImages/static-1.389
 samemodel.php?imageid=*&pg=*&sortcriteria=6
+samsung
 sap/saphana/pilot/
 sap/saphana/pilot/offer/
 sap/saphana/pilot/offer/confirmation
@@ -20190,7 +20347,7 @@ section/3
 section/36
 section/6
 section/8
-section/9http://issuu.com/robots.txt
+section/9http://telegram.me/robots.txt
 sectionized
 sections
 sections$
@@ -20302,7 +20459,6 @@ seo/learn/assets*?topic=
 seokicks
 series
 server
-server-status
 server/banners
 server/body-include
 server/bottom-notes
@@ -20360,6 +20516,7 @@ set_cookie.pl
 setbucket
 setprefs
 settings
+settings
 settings/*
 settings/deactivated
 settings/linkaccount
@@ -20497,6 +20654,7 @@ sharing
 sharing/files
 sharing/folders
 shelf/user_shelves
+shenghuo
 shenghuo/search?
 shifen/
 shihui
@@ -20507,12 +20665,16 @@ shipping-payment
 shippingassistant
 shippinginformation
 shop
+shop$
+shop/
 shop/*&
 shop/browse/overlay
+shop/collections
 shop/global
 shop/iphone/payments/overlay
 shop/search.ep
 shop/view/product
+shop?
 shop_name_search_service
 shop_pins
 shopfront-api
@@ -20567,7 +20729,7 @@ showdependency
 shows/
 showtag
 showtime/.requestinvite.html.swp
-showtimes/location/*http://images-na.ssl-images-amazon.com/robots.txt
+showtimes/location/*http://tripadvisor.com/robots.txt
 shr*
 shutterstock
 si
@@ -20630,6 +20792,7 @@ signup/payment%
 signup/payment?
 signup?
 signup_modal
+signurl
 simple.html$
 simplepost
 simpleshare
@@ -21054,6 +21217,7 @@ social
 social-connect/send
 social/photosui/images
 social/photosui/sitemap
+social/search?
 social/social-security-directory/cai
 social_media
 socialapp/
@@ -21116,6 +21280,7 @@ songs/*/comments
 songs/*/metadata
 songs/*/show_amp_header_play_toggle
 songs/for_profile_page
+sony
 sorry
 sort=
 sortBy=
@@ -21141,6 +21306,7 @@ sp=
 sp_url=
 spade-widget
 spartan
+spbasic
 spbot
 spc-source=
 special-promo/2for1
@@ -21220,6 +21386,7 @@ sports/whatson
 sports_fp_ms
 spp/
 spreadsheet
+spresults
 sprint_wml
 sprint_xhtml
 sq/android/collections
@@ -21305,6 +21472,7 @@ statements
 static
 static-assets/documents/PwnedPasswordsTop100k.txt
 static-assets/images/collection
+static/
 static/*.css$
 static/*.css?
 static/*.gif
@@ -21436,6 +21604,8 @@ storeclick
 storefront
 storepicker
 storerequest
+stores$
+stores/
 stores/page/*visitId=
 stories
 story
@@ -21522,6 +21692,7 @@ subscribe_success
 subscription
 subscription/*?*onboarded=
 subscription/*?*source=
+subscriptioncancel
 subscriptions
 subscriptions/manage/
 subscriptions/securejson/
@@ -21750,7 +21921,7 @@ tag
 tag-manager
 tag-manager-360
 tag/
-tag/*+http://discogs.com/robots.txt
+tag/*+http://amazon.es/robots.txt
 tag/*/a
 tag/*/a/
 tag/*/b
@@ -21832,11 +22003,13 @@ talk/new_topic
 talk/user
 talk/user/*/article
 talks2
+talktalk
 tally-loader-react/static-1.65
 target_baby
 target_group
 targetdirect_group
 tasks
+tastesurvey
 tax-center
 tax.php
 tax_countries.php
@@ -21870,6 +22043,7 @@ tbproxy
 tc/accept
 tdaroundme
 tdautourdemoi
+tdc
 tdid=*
 tdir/p/kids-back-to-school
 tdv2_fp
@@ -21903,6 +22077,7 @@ technology/archive/
 technology/tagged/*page
 telegraph
 telehealth-for-women
+telenor
 television/entretenimiento
 television/teletexto/incibe
 television/teletexto/noticias
@@ -21913,12 +22088,24 @@ television/tve-internacional/asia/?
 television/tve-internacional/europa/?
 television/tve-internacional/parrilla/?
 television/tve-internacional/programacion-tv/?
+telfort
 telsearch
 temp
 temp/
 template
 templateabuse
 templatedata
+templatepackage/3.0/Templates
+templatepackage/3.0/examples
+templatepackage/3.0/templates
+templatepackage/4.0/docs
+templatepackage/4.0/gallery-internal
+templatepackage/Templates
+templatepackage/examples
+templatepackage/gadgets
+templatepackage/modules
+templatepackage/subtopic
+templatepackage/templates
 templates
 templates/
 templates/*/*/?uid=
@@ -21952,6 +22139,7 @@ terms/looker/legal/sccs
 terms/looker/legal/suppliers/partner-assignment-faq
 terms/looker/legal/suppliers/supplier-code-of-conduct
 terms_of_use
+termsofusechanges
 test
 test-2
 test-2$
@@ -21968,9 +22156,9 @@ test5
 test6
 testcases
 testfl
-testhttp://moz.com/robots.txt
+testhttp://use.typekit.net/robots.txt
 testing
-testing/http://facebook.com/robots.txt
+testing/http://accenture.com/robots.txt
 testpage-forhomepage
 tests/*
 tetro
@@ -21978,6 +22166,7 @@ text
 text_edition
 tfemail
 th
+th$
 th*/blog-article-remove-faq-utms-*.js
 th*/cdn-cgi/challenge-platform
 th*/step
@@ -22040,6 +22229,7 @@ th/tools/logo-maker/onboarding
 th/user
 th/users
 th/ventureone
+th?
 th_th/templates/M
 thank-you
 thank-you-enterprise-demo
@@ -22093,6 +22283,7 @@ ti/p
 tickets
 tickets-external?
 tiempo/*p=
+tim
 time/change/*?
 time/change/@
 time/map/*_escaped_fragment_
@@ -22328,12 +22519,13 @@ translate_static
 translate_suggestion
 translate_t
 translation
+translator/?ref=
 transmute/static-1.85
 transmute/static-2.6
 transmute/static-2.8
 transmute/static-2.9
 transparency
-transparency/http://anchor.fm/robots.txt
+transparency/http://ehow.com/robots.txt
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14885
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14895
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14896
@@ -22356,15 +22548,32 @@ travel-vacations
 travel/
 travel/2013/aug/22/been-there-readers-competition?
 travel/8711559/The-Telegraph-Travel-Awards-2011.html
+travel/api
 travel/clk
+travel/css
 travel/entity
+travel/flight-search
+travel/flight/flightSearch
+travel/flight/flightSearchAction
+travel/flight/search/?
+travel/flight/search?
+travel/flights
 travel/flights/s
+travel/hotel-search
 travel/hotel/e
+travel/hotel/hotelMiniSearchRequest
+travel/hotel/hotelSearch
+travel/hotels/checkout
+travel/hotels/detail
 travel/hotels/entity
 travel/hotels/hotel-finder
 travel/hotels/hotel-finder/$
+travel/hotels/search/?
+travel/hotels/search?
 travel/hotels/stories
+travel/scripts
 travel/search
+travel/secure
 travelnevadahighway95tc
 treasury
 treasury/treasdocuments
@@ -22420,6 +22629,7 @@ trylater
 tsa
 tto
 tucritica
+tudum
 tune*retpath=
 turbo
 turbo*?*ajax=1
@@ -22480,6 +22690,7 @@ tve/b/zonadisney
 tve/programo
 tve/teletexto
 tvschedule
+tvsignup
 tw
 tw*/blog-article-remove-faq-utms-*.js
 tw*/cdn-cgi/challenge-platform
@@ -22525,7 +22736,6 @@ tw/ventureone
 tweet
 twitter
 twitter/ptt.php
-tx_form_formframework
 txt
 ty$
 ty?
@@ -22534,9 +22744,6 @@ typ?
 typeahead
 typo
 typo3
-typo3conf
-typo3conf/ext
-typo3temp
 u
 u.cfm
 u.php
@@ -22790,6 +22997,7 @@ unternehmen/o
 unternehmen/suche
 unterstuetzer.html
 until=
+upcomingevents
 update-your-browser
 update.php
 update_
@@ -22829,6 +23037,7 @@ ur/search
 ur/user
 ur/users
 urchin_test
+url
 url-shortner
 urlinator/static-1.54
 urlinator/static-1.55
@@ -23245,6 +23454,7 @@ verify
 verify_age
 verify_your_email$
 verify_your_email?
+verifyidentity
 version
 versions
 vet/addl/covid19partners
@@ -23357,13 +23567,31 @@ videodash.cms*
 videodetails
 videorequest
 videos
+videos/?
 videos/assets
+videos/browsing
 videos/e
+videos/explore
+videos/favorites
+videos/feed
+videos/feedcontent
 videos/get_video_by_uid/
 videos/images
 videos/js
+videos/music
+videos/music/
+videos/music/?
+videos/music?
+videos/riverview/relatedvideo
+videos/riverview/relatedvideo/?
+videos/riverview/relatedvideo?
 videos/search.php
+videos/search/?
+videos/search?
+videos/trending
+videos/trendingcontent
 videos/viewmeta/
+videos?
 videosearch
 view
 view-3d-home
@@ -23376,6 +23604,7 @@ view_item.pl
 view_message.pl
 viewconfig$
 viewer
+viewingactivity
 viewlocale=zh_CN
 vieworder
 viewrecentepisode
@@ -23384,6 +23613,7 @@ viitem
 vilos
 vinterdev/
 vip
+virginaustralia
 virtual
 virtualization/
 virtualsupportassistant
@@ -23432,6 +23662,8 @@ vn/tools/business-name-generator/searchutf8
 vn/tools/logo-maker/onboarding
 vn/ventureone
 vod
+vodafone
+vodafonedemobilelaunch
 voices/
 volunteer/
 vote
@@ -23613,9 +23845,14 @@ whichmba/forum
 whichmba/webinars?
 whichmba/webinars?page=
 whsmiths
+whysecure
 whythisad
 widget
 widget-landing-page.html
+widget/cr
+widget/entity/search/?
+widget/render
+widget/snapshot
 widget_auth.php
 widget_chunks
 widgets
@@ -24405,6 +24642,7 @@ word/embed
 wordpress/wp-admin
 wordpress/wp-admin/admin-ajax.php
 work
+work
 work/_
 work/acceptinvitation
 work/account
@@ -24502,8 +24740,8 @@ wp-admin
 wp-admin/
 wp-admin/admin-ajax.php
 wp-admin/admin-ajax.php
-wp-admin/admin-ajax.phphttp://aljazeera.com/robots.txt
-wp-admin/admin-ajax.phphttp://fool.com/robots.txt
+wp-admin/admin-ajax.phphttp://tiff.net/robots.txt
+wp-admin/admin-ajax.phphttp://umc.org/robots.txt
 wp-admin/load-scripts.php
 wp-admin/load-styles.php
 wp-adv
@@ -24577,7 +24815,7 @@ wp-content/uploads/
 wp-content/uploads/wc-logs
 wp-content/uploads/woocommerce_transient_files
 wp-content/uploads/woocommerce_uploads
-wp-content/uploads/wp-import-export-lite/http://rebrand.ly/robots.txt
+wp-content/uploads/wp-import-export-lite/http://woodgears.ca/robots.txt
 wp-dyn
 wp-includes
 wp-includes*.css
@@ -24637,12 +24875,14 @@ x
 x/oauth
 x?
 xb/en/
+xboxone
 xbp/search
 xc/
 xclick-auction
 xcs0MhRsfxHG.js
 xd/en/
 xdoor
+xfinity
 xfx7
 xh/forums
 xh/kb/*/discuss
@@ -24673,6 +24913,7 @@ xmlsearch
 xoonboarding
 xoplatform/logger/api/logger
 xorouter
+xs4all
 xsl/
 y1=
 yaca
@@ -24706,8 +24947,10 @@ your
 your-account
 your-orders
 your-shop
+yourAccount
+youraccount
 youreurope/businessACC
-youreurope/businessDEV/http://europarl.europa.eu/robots.txt
+youreurope/businessDEV/http://money.cnn.com/robots.txt
 youreurope/citizensACC
 youreurope/citizensDEV
 yours
@@ -24914,6 +25157,7 @@ zh/ventureone
 zh_CN
 zh_cn
 zh_tw/templates/M
+ziggo
 zite-importer
 zoeken$
 zoeken.html$
diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt
index 5cf9999f..a8bd8251 100644
--- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt
+++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt
@@ -1092,6 +1092,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 .ics$
 .inc$
 .inc$
+.ini$
 .jhtml
 .jhtml$
 .jpeg
@@ -4569,6 +4570,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /checkin
 /checklist$
 /checkout
+/checkout
 /checkout/
 /checkout/*
 /child
@@ -6003,6 +6005,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /login*/search?
 /login-redirect
 /login.cms*
+/login/
 /login/*
 /login/verify-email
 /login?
@@ -6191,6 +6194,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /muslim-sexy-videos.html
 /mvopinions/*
 /my
+/my-account
 /my-account/profile
 /my-account/register/newsletters-setup
 /my-account/register/profile-setup
@@ -6756,6 +6760,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /reg/regmsg.xhtml
 /register
 /register$
+/register/
 /register/*
 /register/?
 /registration$
@@ -8378,6 +8383,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?bclid=
 ?bcpid=
 ?bctid=
+?begin
 ?benderPackage=
 ?beta=
 ?betaplayer=
@@ -8856,7 +8862,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?mobileset=
 ?mode
 ?mode*
-?mode=
 ?model=
 ?modelNumber=
 ?module=
@@ -9062,7 +9067,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?q=getImageNew
 ?q=login_page
 ?q=logout
-?q=logout/
 ?q=node/*/edit
 ?q=node/*/nodequeue
 ?q=node/add
@@ -9667,10 +9671,12 @@ Account/*
 Account/Register
 Account/Register.aspx
 Account/register.aspx
+AccountAccess
 AccountActivityView
 AccountBalanceView
 AccountDisavow?
 AccountMerge
+AccountStatus
 Account_Billing
 AccurintForCollectionsLP
 Achievements
@@ -9784,6 +9790,7 @@ App_config/
 AppleMapsAction
 Applications
 Apps/
+Arabic
 Architext
 Archive/New_Records_Testing
 Archive/PrintMaterial/ACS_Journal/1/files
@@ -10710,6 +10717,7 @@ Bender
 Benoit/Public/DVDDB
 BetterBySea
 Biker
+BillingActivity
 BillingAndPayments
 BillingDetail
 BillingInvoiceFile
@@ -10791,6 +10799,7 @@ CampaignConfiguration
 CampaignId
 CampaignManagement
 CancelBooking
+CancelPlan
 CaptchaValidationLog
 Cargo-Webdesign-Directory
 Carrello
@@ -10811,6 +10820,7 @@ Celebrate100Million
 CertificateOfExcellenceRequestSupport
 ChangeEmailAddressView?
 ChangePassword
+ChangePlan
 ChangeSecurityInfoView
 ChatGPT
 ChatGPT-User
@@ -10831,6 +10841,7 @@ CheckoutOrderBillingView
 CheckoutOrderShippingView
 CheckoutPayInStoreView
 CheckoutSignInView
+Chinese
 Christian/Books/WISHLIST
 Christian/Books/backorder_email
 Christian/Books/bookbag
@@ -10855,7 +10866,6 @@ Civ
 Claim
 Classifieds/
 ClientAuth
-ClientBin/
 ClientLink
 ClientLogin
 ClientStorageProxy.aspx
@@ -11022,6 +11032,7 @@ DisplayCmd
 DisplayMap
 DisplayView
 Divisions
+DoNotTest
 Doc
 Doc-rm
 Doc-rss
@@ -11072,6 +11083,7 @@ ET/
 EULegal
 EarlyRequesterJS/static-1.38
 EdadsV2
+EditProfiles
 Editorial/
 Education/SchoolHelp.htm
 Education/SchoolStudents/Olympiad/paper2011.asp
@@ -11083,6 +11095,7 @@ EmailCartView
 EmailHotel
 EmailMap
 EmailMemberRegistration
+EmailPreferences
 EmailSpellCheckJson
 Emailer
 Emailv_v2
@@ -11344,6 +11357,7 @@ FavoriteReviewManagement
 Feature
 Featured
 FeaturedShowMoreOverlay
+Features/SpinaBifidaProgram/
 Feedback/
 Feeds
 Fetch
@@ -11360,6 +11374,7 @@ FinancialServices
 Find*
 FindRestaurants
 FinishAppInstall
+FireTV
 FiveSteps/
 FiveStepsToHealth/
 Flamingo_SearchEngine
@@ -11515,6 +11530,7 @@ HX62
 HYx10rg3
 HandleHotelSuperPortal
 Handlers
+HdToggle
 HeadJS/static-2.115
 HeadJS/static-2.83
 HeadJS/static-2.84
@@ -11552,6 +11568,7 @@ HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
 How_To_Get_Sky
+HpImageArchive.aspx
 HubStyle/static-2.309
 HubStyle/static-2.313
 HubStyle/static-2.315
@@ -11747,6 +11764,7 @@ KeyFeatures/
 Kimengi
 Kimengi/nineconnections.com
 Knowledge-Search
+Korean
 Krypto
 LA/
 LB/
@@ -11775,6 +11793,7 @@ Labs/
 LanderLogger
 LandingPage
 Landingpages.html
+LanguagePreferences
 Languages/Pages/Volunteer-Stipend.aspx
 Languages/SiteCollectionDocuments/FAQ-County-Translations.pdf
 Large-Files/
@@ -11832,10 +11851,12 @@ Login-OAuthReentryFacebook*
 Login.html
 Login/
 Login?
+LoginHelp
 Logoff
 LogonForm
 LogonForm
 LogonFormView
+Logout
 Logs
 MA/
 MAINTAINERS.txt
@@ -11908,11 +11929,13 @@ MailHelpR1
 MakePaymentView
 Manage/
 ManageAccountAccess
+ManageDevices
 ManageListing
 ManageOrder
 ManageOwnersTopReasonsApi
 ManagePaymentAccountView
 ManagePhotos
+ManageProfiles
 ManageReturns
 ManageShopListingApi
 ManageShopping
@@ -11933,6 +11956,7 @@ Marketing/EcapSurvey.aspx
 Marketplace/
 MasterPages
 MathJax
+Maxis
 Maxthon
 McKinsey
 MedCenter
@@ -12020,6 +12044,7 @@ MemberTravelMap
 Membership/Memberzone/
 Membership/e-Membership/app-help.asp
 Membersites
+Meo
 Merchant2
 MessageRunDetailID=
 Messages
@@ -12179,6 +12204,7 @@ Moreover
 Moreoverbot
 Moved
 Movies
+MoviesYouveSeen
 MptUrl
 Ms.
 Msgs/
@@ -12253,6 +12279,7 @@ NetSeer
 NewMessage
 NewOwners
 NewTopic
+NewWatchInstantlyRSS
 News
 News/Flash.aspx/
 News/Index
@@ -12423,6 +12450,7 @@ Pages/Search.aspx
 Pages/myetocsunsubscribe.aspx
 Pages/subprocessors.aspx
 PageviewRecord
+Panasonic
 Panier
 PanoramaXml
 Participate
@@ -12476,6 +12504,7 @@ PoiReviewDetail
 PoiTipDashboard
 Politics/qGhqEUxkwY1XCf44StpdPM/Asaram-Bapu-From-building-an-empire-of-Rs-10000-crore-to-b.html
 Politics/qGhqEUxkwY1XCf44StpdPM/Asaram-Bapu-From-building-an-empire-of-Rs-10000-crore-to-b.html?facet=amp&utm_source=googleamp&utm_medium=referral&utm_campaign=googleamp
+PopupDetails
 Portal/
 PortalIdParser/static-1.27
 PortalIdParser/static-1.28
@@ -12584,6 +12613,7 @@ Products/tv/androidtv
 Products/tv/seeingisbelieving
 Profile
 Profile/ViewPdf
+ProfilesGate
 Programmes/Search-Courses
 Programs
 Project
@@ -12598,6 +12628,7 @@ PropertyLayer/
 Provider/Image/
 Provider/PhysicianRatings/
 Providers/
+Proxy.ashx
 ProxyToolsAccountSettingsTool
 ProxyToolsBillingAndPaymentsTool
 ProxyToolsBillingDetailTool
@@ -13032,6 +13063,7 @@ Solr
 Soltango/de-DE/
 Soltango/en-GB/
 SolutionsDisplay
+Sony
 SortAttractions
 SortEateries
 SortHotels
@@ -13170,6 +13202,15 @@ TeleportPro
 Telerik.Web.UI.WebResource.axd?_TSM_HiddenField
 Telerik.Web.UI.WebResource.axd?compress
 Template/
+TemplatePackage/3.0/Templates
+TemplatePackage/3.0/examples
+TemplatePackage/4.0/docs
+TemplatePackage/4.0/gallery-internal
+TemplatePackage/Templates
+TemplatePackage/examples
+TemplatePackage/gadgets
+TemplatePackage/modules
+TemplatePackage/subtopic
 Templates
 Templates/
 Templates/*
@@ -13476,6 +13517,7 @@ WelcomeView
 WhatsNearby
 Wholesale-*-c-*-0-1-*.html
 Wholesale-Attribute-*.html
+WiViewingActivity
 WidgetBrowserExtension
 WidgetEmbed
 WidgetListAsync.naver
@@ -13514,6 +13556,8 @@ YU/
 Yandex
 Yasni
 YfcuVaVBkfnzl2j6UbEcK0Oiqkv88jx6w
+YourAccount
+YourAccountPayment
 Youtube
 ZA/
 ZCestino
@@ -13669,6 +13713,7 @@ _blank
 _blog_rss
 _bm/*
 _borders
+_borders
 _buildManifest.js$
 _cam-*_tab-*.html
 _catalogs/
@@ -13710,6 +13755,7 @@ _custom/*
 _deactivate.asp
 _debug/
 _derived
+_derived
 _design
 _dm
 _docs
@@ -13748,6 +13794,7 @@ _feeds
 _finance_doubledown
 _fparam-*_viewport-*.html
 _fpclass
+_fpclass
 _fragment
 _fragment?
 _frdtcr
@@ -13859,6 +13906,7 @@ _old_design
 _openstat
 _openstat
 _overlay
+_overlay
 _ows_includes/php/
 _ows_includes/temp/
 _p
@@ -13948,6 +13996,8 @@ _test
 _testing/
 _th_
 _themes
+_themes
+_tmp
 _translations?
 _tsc
 _uac/adpage.html
@@ -13970,6 +14020,7 @@ _vti_inf.html
 _vti_log
 _vti_log
 _vti_map
+_vti_map
 _vti_pvt
 _vti_pvt
 _vti_txt
@@ -14357,6 +14408,8 @@ academia/books/search.html
 academic
 academic-subscriptions
 academic/
+academic/profile
+academic/search
 academicaffairs/commencement/book
 academics/archive
 academics/college/special-programs/public-service-old
@@ -14438,9 +14491,7 @@ account/default.aspx
 account/diary-settings
 account/edit
 account/emailoptout/?*token=
-account/en-ca/email-sign-up
 account/forgot-password
-account/fr-ca/abonnement-courriels
 account/invite
 account/login
 account/logout
@@ -14507,6 +14558,7 @@ account_settings/set_password?
 account_settings/set_your_password$
 account_settings/set_your_password?
 account_settings?
+accountaccess
 accounthelp
 accountinfo.php
 accounts
@@ -14538,6 +14590,7 @@ accounts/new
 accounts/o8
 accounts/o8/id
 accounts/unlock
+accountstatus
 acct
 acctmgmt/*
 acd
@@ -14559,6 +14612,7 @@ acis/whatsnew.html
 acjohnston
 acl
 aclayton
+aclick
 aclk
 aclp-vle-decom
 acmendez
@@ -14857,6 +14911,7 @@ adelrio
 adev
 adfox
 adframe
+adfs/
 adg
 adg321
 adgranillo
@@ -15081,7 +15136,6 @@ adwordsresellers
 adx/bin
 adzikiewicz
 ae
-ae-en/
 ae/
 ae/en/book-a-trip/flights/fare-types/fare-guide-information.html
 ae/en/contact/search?param=
@@ -15271,6 +15325,7 @@ aiprojecthks
 aircraft-icons/sprite
 airport/
 airport_transfer_deeplink
+airtel
 aisharma
 aislingibson
 aj_windows/travelagencies
@@ -15483,6 +15538,7 @@ aliciademaio
 aliciafroio
 alikhani
 alilly
+alink
 aliraja
 alireza.haghighi
 alisamanning
@@ -15656,6 +15712,7 @@ amp-iframe
 amp-proxy
 amp-social-login
 amp-tealium
+amp/
 amp/*/more.html
 amp/*/page-*/$
 amp/amp/*
@@ -15766,6 +15823,7 @@ analyzer
 analyzer-api
 anamikasaha
 anamoss
+anan
 anapi
 anass-mourjane
 anastaciakay
@@ -15904,6 +15962,7 @@ anon-ftp
 anon/AdPreview$
 anons_test.php
 anonymous
+anpe
 ans
 anselmocassiano
 anshumanc
@@ -16103,6 +16162,7 @@ api/jobPostings/jobs
 api/latest-articles
 api/legacy/pse
 api/legendary-data/relay
+api/maps/
 api/meerkat/page
 api/messages
 api/metainfo/tv
@@ -16543,7 +16603,6 @@ ar-eg/televisionat-bahs-sawti-men-doun-istikhdam-al-aydi
 ar-eg/televisionat-dolby-vision-atmos
 ar-eg/wade3-mou3ayar-men-netflix-li-televisionat-zakia
 ar-en
-ar-es/
 ar-es/shop/*?*&*&*&*
 ar-iq/
 ar-iq/home-theatre-sound-bars/neckband-speakers
@@ -17218,7 +17277,6 @@ at*/people/*/circle
 at*/people/*/favorites
 at*/shop/*/favoriters
 at*/shop/*/sold
-at-de/
 at-en
 at/
 at/*?order=
@@ -17443,7 +17501,6 @@ au*/shop/*/sold
 au*/step
 au*/submit
 au*/submitted
-au-en/
 au-en/shop/*?*&*&*&*
 au-fr
 au.html
@@ -17739,6 +17796,7 @@ audreytiew
 audreywoz
 augustin_guyonnet
 augustyh
+aui/inbound
 aura
 aura/promo
 auroregonzalez
@@ -17814,7 +17872,6 @@ authorizemessaging.html
 authormerge.php
 authors
 authors.pl
-authors/
 authors/*
 authors/*page=
 authors/57/james-rondeau
@@ -17842,6 +17899,7 @@ autocredit/sber_landing
 autodiscover/
 autokolcsonzes/eredmenyek
 autolist
+autologin
 automobile/recherche.php
 autonoleggio/risultati
 autonvuokraus/tulosta
@@ -18523,7 +18581,6 @@ bars
 baruchkrauss
 barzahlung.html
 bas
-base/
 base/login
 base/reportbadoffer
 base/s2
@@ -18752,7 +18809,6 @@ be-fr*/cdn-cgi/challenge-platform
 be-fr*/step
 be-fr*/submit
 be-fr*/submitted
-be-fr/
 be-fr/*?*shpxid=
 be-fr/*digital_wallets/dialog
 be-fr/admin
@@ -18785,7 +18841,6 @@ be-fr/tools/business-name-generator/searchpage
 be-fr/tools/business-name-generator/searchutf8
 be-fr/tools/logo-maker/onboarding
 be-fr/ventureone
-be-nl/
 be/
 be/*?*shpxid=
 be/*?order=
@@ -19053,6 +19108,7 @@ bfigueroa
 bfitzgerald
 bfl
 bfmp
+bfp/search
 bfrank
 bg
 bg-*/
@@ -19317,6 +19373,7 @@ bin/sleep.cgi
 bin/ti
 binary-data/DIY
 binary-data/EGALLEY
+bing-site-safety
 bingbot
 bingsiteauth.xml
 bingwang
@@ -19608,7 +19665,6 @@ blog/category/*/post/latest.rss
 blog/category/*/post/latest.xml
 blog/category/ghostery-news
 blog/cgi-bin
-blog/comment
 blog/comments
 blog/coverage-for-topic/
 blog/credit-card-reviews
@@ -19778,6 +19834,7 @@ blogs/practitioners-of-the-month
 blogs/product
 blogs/research/search
 blogs/search
+blogs/search/
 blogs/slow-ride
 blogs/tag
 blogs/technology.atom
@@ -20294,6 +20351,7 @@ boutique/sitemaps
 boutiques
 boutiques/la-rentree-des-sportifs
 boutiques/la-rentree-des-sportifs$
+bouyguestelecom
 bova/
 bowles
 box-week
@@ -20546,6 +20604,7 @@ brinda
 brink-roby
 brinton
 brisasarai
+britishairways
 broadband
 broadway-revere/appointments-and-referrals/appointment-thankyou
 brodyzt
@@ -20584,6 +20643,7 @@ browse/devicesetup
 browse/domain
 browse/download
 browse/feed
+browse/genre
 browse/homes/all
 browse/issue/*/follow
 browse/laboratory
@@ -20616,6 +20676,7 @@ brt
 bruce
 brucker
 bruckerjoseph
+brutus
 brw
 bryan
 bryanhsu
@@ -21261,7 +21322,6 @@ c4p_log/
 c87a07c1314d07aa13275a98667ef25da5633259.html
 c?*attr_*=
 cDN_contENt
-cHash
 c_amit
 ca
 ca*/blog-article-remove-faq-utms-*.js
@@ -21278,7 +21338,6 @@ ca-ES/*barcelona
 ca-ES/*spain
 ca-ES/office-space
 ca-ES/signup
-ca-en/
 ca-es/
 ca-fr*/blog-article-remove-faq-utms-*.js
 ca-fr*/cdn-cgi/challenge-platform
@@ -21290,7 +21349,6 @@ ca-fr*/shop/*/sold
 ca-fr*/step
 ca-fr*/submit
 ca-fr*/submitted
-ca-fr/
 ca-fr/*?*shpxid=
 ca-fr/*?order=
 ca-fr/*color=
@@ -22068,6 +22126,8 @@ card_management.php
 cardauth
 cardealers/partial/dealerdetails-about.xhtml
 cardealers/partial/photoMedia.xhtml
+cardedeu%40thesaurus.net
+cardedeu@thesaurus.net
 cardfinderservice/
 cardinals/fans/text-messaging-terms-and-conditions
 cardinals/search
@@ -24826,7 +24886,6 @@ celt/users/international/overseas.htm
 celtics/node
 celtics/users
 celtweb/OAI/OAI-collaborative-institutions
-cemetery/*/memorial-search
 cemtest2
 census
 centenary-fund
@@ -25023,8 +25082,6 @@ ch*/people/*/circle
 ch*/people/*/favorites
 ch*/shop/*/favoriters
 ch*/shop/*/sold
-ch-de/
-ch-fr/
 ch/
 ch/*?order=
 ch/*color=
@@ -25131,6 +25188,7 @@ change-shipping
 changeLanguage
 change_password=
 changebrowser
+changeplan
 changes
 changes-that-qualify-for-special-enrollment/
 changliu
@@ -25451,6 +25509,7 @@ childrenCount=
 childtaxcredit
 chilton
 chimileski
+chinese
 chinese/about/school-staff.php?id=ODAwMjIy&page_var=personal
 chinese/about/staff-by-role.php?id=ODAwMjIy&page_var=personal
 chineselanguageseries
@@ -25613,7 +25672,6 @@ ckgeiger
 ckorponay
 cksum=
 cl
-cl-es/
 cl-es/shop/*?*&*&*&*
 cl/
 cl/Download
@@ -26320,6 +26378,7 @@ comeuntochrist/*/internal-use-only/
 comeuntochrist/*/search-results/
 comeuntochrist/lp/
 comeuntochrist/teach/
+comhem
 comic-indies
 comic-indies/$
 comic-indies/about
@@ -26453,7 +26512,7 @@ community/*/latest$
 community/*/live
 community/*/post
 community/*/post-
-community/*/profile/*http://cointelegraph.com/robots.txt
+community/*/profile/*http://10250263.fls.doubleclick.net/robots.txt
 community/*/rss
 community/*/topics
 community/*/trackback
@@ -26528,6 +26587,7 @@ comodo-vs-symantec/?
 comp
 comp/
 compagnie
+companies
 companies/
 companies/*/edit
 companies/*/embed
@@ -26742,6 +26802,7 @@ concrete/*.png
 concrete/*.svg
 concrete/*.xml$
 concursocampusparty/files
+concussion/HeadsUp/training/
 condiciones-de-uso
 conditions-diseases?f*
 conditions-diseases?linkId*http://chor.helbling-verlag.de/robots.txt
@@ -26863,6 +26924,7 @@ consult
 consultancies
 consultas?filter=past&geozone_id
 consultation-startup?*
+consultationoffres
 consultations?
 consumer-profile
 consumer-tools/educator-tools/youth-financial-education/curriculum-review/tool
@@ -27004,6 +27066,7 @@ content/*/schedule
 content/*/synchronous_inspection
 content/*/thenables
 content/*/util
+content/*http://theculturetrip.com/robots.txt
 content/README.txt
 content/aarpe/en/home/
 content/aarpe/es/home/
@@ -27852,6 +27915,7 @@ courses/section
 courses/transcript
 courtneytwittekind
 courts
+courts
 cover
 cover-it-live/
 cover-photo.jspa
@@ -27908,6 +27972,7 @@ cpspa
 cpx.php
 cqiu
 cr
+cr$
 cr/
 cr/Download
 cr7
@@ -27917,6 +27982,7 @@ cr7/cgi-bin
 cr7/includes
 cr7/logs
 cr7/tmp
+cr?
 craft
 crafts
 crai/big
@@ -28354,6 +28420,7 @@ cservice/ya_index.gsp
 cshalaby
 csi_204
 csilvaportero
+csl
 csmlists/addbookmark
 csmlists/topic
 csn-search-app/*
@@ -28446,7 +28513,6 @@ css/site.sections.careers.peoplebank.css
 css/site.sections.multimedia.css
 css2/
 css=*
-css?
 css?*
 csswg/report
 cstudio
@@ -28654,7 +28720,6 @@ cz*/step
 cz*/submit
 cz*/submitted
 cz-CZ
-cz-cs/
 cz-cz/
 cz-en
 cz/
@@ -29996,6 +30061,12 @@ demo-product
 demo-request/
 demo-video
 demo-videos
+demo/closed-unclosed-performance
+demo/closed-unclosed-performance?closed
+demo/safari-reader-test-1
+demo/safari-reader-test-2
+demo/safari-reader-test-3
+demo/safari-reader-test-nielshttp://mathworks.com/robots.txt
 democenter
 demokratie-jetzt.html
 demonstration
@@ -30197,6 +30268,7 @@ device-prefs
 device/activate
 device/tracker/
 deviceList
+deviceManagement
 devinmccabe
 devlabs/
 devladervishmahler
@@ -30901,7 +30973,6 @@ distasio
 distribution
 distribution-terms.asp*
 distribution/
-distributions
 distro.html
 divany-i-kresla
 divers/humour/amp
@@ -30917,7 +30988,6 @@ dk*/cdn-cgi/challenge-platform
 dk*/step
 dk*/submit
 dk*/submitted
-dk-da/
 dk-da/Brancher/PI-materialer/
 dk-da/Brancher/pi-materialer/Documents/infrastruktur/
 dk-da/l%C3%B8sninger/Fakturering%20og%20opkr%C3%A6vning/Leverand%C3%B8rservice/Pages/%A6ndring.aspx
@@ -31159,7 +31229,6 @@ dnicholson
 dnigrin
 dnn
 do
-do-not-crawl
 do/
 do=getLastComment?
 do=getNewComment
@@ -31945,6 +32014,7 @@ editorial/sports/$
 editors
 editorspicks
 editorv2_shop_aux/download_course_student_certificate
+editpayment
 edits
 edmontonalberta
 edoardoteso
@@ -32008,7 +32078,6 @@ edwardclapp
 edwardcliff
 edwardii-conference
 ee
-ee-en/
 ee/
 ee/Download
 ee/forums
@@ -32255,6 +32324,7 @@ elim
 eliminar-busqueda
 eliminar-favoritos
 elio
+elisa
 elisanew
 elisarubbo
 elisewilkes
@@ -32367,6 +32437,7 @@ emails/verify
 emailservice
 emailsignup
 emailsignup/
+emailunsubscribe
 emailv_v2
 emallove
 embargo
@@ -32565,7 +32636,6 @@ en-CA/firefox/collections/4757633/$
 en-CA/firefox/collections/mozilla
 en-CA/firefox/collections/mozilla/$
 en-CA/firefox/search
-en-CA/order-status
 en-CH/
 en-CN
 en-DA
@@ -32790,29 +32860,11 @@ en-ca*
 en-ca.www.cloudflare.com
 en-ca/
 en-ca/*
-en-ca/Basket
-en-ca/Search
-en-ca/about/
-en-ca/basket
-en-ca/brand-hub/
-en-ca/brand/
-en-ca/category/
-en-ca/create-review/
-en-ca/event/
 en-ca/feedback
-en-ca/flyer
-en-ca/help/
 en-ca/lp
 en-ca/mega-nav
-en-ca/order-status
-en-ca/product-sellers/
-en-ca/product/
-en-ca/product/*?icmp=*
-en-ca/search
 en-ca/search?
 en-ca/searchresults
-en-ca/services/
-en-ca/subscription-packages
 en-cd/dolby-vision-atmos-tvs
 en-cd/electronics/how-to-select-tv-by-what-you-watch-gaming
 en-cd/electronics/how-to-select-tv-by-what-you-watch-netflix
@@ -34351,12 +34403,16 @@ enterprise/products/checks
 enterprise/products/checks/index.jsp
 enterprise/seo
 enterpriseform/
+entertain
 entertainment/article/Minerva-s-horoscope
 entertainment/bravo
 entertainment/index.html
 entertainment/test$
 enterzipcode.php
 entities
+entities/search
+entityexplore$
+entityexplore?
 entityreference
 entp
 entr/Die%20Geh%C3%A4lter%20wurden%20gek%C3%BCrzt
@@ -34378,6 +34434,7 @@ entry/comment
 entry/image
 entry/jsonlite
 entrypoint_ndp
+entrytrap
 enus/
 enviar_amigo
 enviarnoticia.aspx
@@ -35467,8 +35524,10 @@ esntial.htm
 esoft-job
 esotiri
 espace-securise/shield
+espace_candidat
 espaceabonne
 espaceabonne/creer-un-compte/$
+espacecandidat
 espanol
 espanol/busqueda/
 espanol/capable-suvs
@@ -35734,6 +35793,7 @@ eucistron
 eugenchua
 eugeniah
 eugenio
+eula
 eula.html
 eum
 eumcollector/beacons/browser/v1
@@ -36305,7 +36365,6 @@ f/*/widget
 f/healthcheck
 f/s/flurry
 f/smarty
-f1.
 f1/
 f1racing
 f7
@@ -36522,6 +36581,7 @@ fckeditor
 fcollado
 fcyrraci
 fd-fr/
+fd/
 fdbk
 fdias
 fding
@@ -37060,7 +37120,7 @@ find/*?official=true
 find/*?page
 find/*?shipping=
 find/*?{
-find/*http://freemius.com/robots.txt
+find/*http://freemusicarchive.org/robots.txt
 find/3DImage.jsp
 find/?srsltid
 find/ApplecareDetails.jsp
@@ -37114,6 +37174,7 @@ firefox/?
 firefox/downloads
 firefox/new_tab_learn_more
 firehose.pl
+firetv
 firm_webinars
 firmaDuzeltKaydet
 firme/pierluigi-battista_80
@@ -37245,6 +37306,7 @@ flow
 flower
 flows
 fls=0
+flu/espanol/
 fluege/cn/*
 fluege/ct
 fluff
@@ -38113,7 +38175,6 @@ fr-BR
 fr-BY
 fr-CA/*canada
 fr-CA/*montreal--QC
-fr-CA/ma-commande
 fr-CA/office-space
 fr-CA/signup
 fr-CN
@@ -38224,28 +38285,11 @@ fr-bj/tv-avec-recherche-vocale-mains-libres
 fr-ca
 fr-ca*
 fr-ca/
-fr-ca/Chercher
-fr-ca/Panier
-fr-ca/a-propos/
 fr-ca/about-us/careers/job-description.html$
-fr-ca/aide/
-fr-ca/categorie/
-fr-ca/chercher
-fr-ca/circulaire
-fr-ca/ecrire-evaluation/
-fr-ca/evenement/
-fr-ca/forfaits-de-abonnement
 fr-ca/ipns
 fr-ca/login-messages
-fr-ca/ma-commande
-fr-ca/marque/
-fr-ca/marques/
 fr-ca/oicms
-fr-ca/panier
-fr-ca/produit/
 fr-ca/ressources/search
-fr-ca/services/
-fr-ca/vendeurs-de-produit/
 fr-cd/electronics/comment-choisir-son-televiseur-par-contenu-jeux
 fr-cd/electronics/comment-choisir-son-televiseur-par-contenu-netflix
 fr-cd/electronics/comment-choisir-son-televiseur-par-contenu-smart-tv
@@ -39029,6 +39073,7 @@ freehosting.php
 freejoin
 freeland
 freemantest
+freemonth
 freepublications/
 freertos/archive
 fren/volumeters
@@ -39110,6 +39155,8 @@ fullscreen
 fullscreen/ReportLinker
 fullwiley
 fun
+fun/api/
+fun/g/
 functions
 funderportal
 funding-for-researchers/our-funding-schemes?*
@@ -39711,6 +39758,7 @@ geomag/WMM/data
 geomag/cdroms
 geomag/data
 geomag/pmag/datafiles
+geooverride
 georganteli
 georgeblaustein
 georgehana
@@ -40300,7 +40348,6 @@ gq/
 gql
 gr
 gr-
-gr-el/
 gr/
 gr/aboutus
 gr/cardholderservices
@@ -41275,7 +41322,6 @@ hk-en*/shop/*/sold
 hk-en*/step
 hk-en*/submit
 hk-en*/submitted
-hk-en/
 hk-en/*?*shpxid=
 hk-en/*?order=
 hk-en/*?services
@@ -41357,7 +41403,6 @@ hk-en/transactions
 hk-en/treasury
 hk-en/ventureone
 hk-en/your
-hk-zh/
 hk-zh/shop/*?*&*&*&*
 hk/
 hk/*?*shpxid=
@@ -41723,6 +41768,7 @@ hotel/*/?type
 hotel/h
 hotel/index.hi.
 hotel/redirect
+hotel/reviews?
 hotel/s
 hotel/search
 hotel/us/the-airstream-van.*.html
@@ -41773,6 +41819,7 @@ hotels/reservation-payment.htm
 hotels/rpc
 hotels/s
 hotels/search
+hotels/search
 hotels/search-results.htm
 hotels/search/list*
 hotels/search/map*
@@ -41837,6 +41884,8 @@ hp/focus-retrocompatibile$
 hp/video
 hp_channels
 hpan
+hpm
+hpmob
 hpols_catalog/
 hpws/max-connections.php
 hpws/parallel-downloads.php
@@ -44871,7 +44920,6 @@ html/E98455_01
 html/E99994_01
 html/E99997_01
 html/E99998_01
-html/EN
 html/F10012_01
 html/F10240_01
 html/F10249_01
@@ -44966,10 +45014,8 @@ html/Q77771_01
 html/abonnements
 html/air/routemap
 html/conditions/infos_abos_boutique.jsp
-html/en
 html/favori
 html/ipad
-html/it
 html/js/editor
 html/js/editor/fckeditor
 html/map_delivery_msk.html
@@ -44977,7 +45023,6 @@ html/map_delivery_spb.html
 html/media/pdf
 html/misc
 html/mobile/iPhone2012
-html/node
 html/pdf
 html/salon-2727.pdf
 html/share/rs/rs.php
@@ -45015,12 +45060,11 @@ http://1731177.fls.doubleclick.net/robots.txt
 http://1password.com/robots.txt
 http://2224480.fls.doubleclick.net/robots.txt
 http://4shared.com/robots.txt
-http://9665741.fls.doubleclick.net/robots.txt
+http://6323411.fls.doubleclick.net/robots.txt
 http://abc7chicago.com/robots.txt
 http://abc7ny.com/robots.txt
 http://abcnews.go.com/robots.txt
 http://abduzeedo.com/robots.txt
-http://aclu.org/robots.txt
 http://ad.doubleclick.net/robots.txt
 http://adcell.de/robots.txt
 http://adeje.es/robots.txt
@@ -45031,7 +45075,7 @@ http://amazon.com.mx/robots.txt
 http://amazon.de/robots.txt
 http://amazon.in/robots.txt
 http://amazon.jobs/robots.txt
-http://ampmexterminators.com/robots.txt
+http://amzn.to/robots.txt
 http://androidpolice.com/robots.txt
 http://antena3.rtp.pt/robots.txt
 http://api.ning.com/robots.txt
@@ -45040,6 +45084,7 @@ http://apod.nasa.gov/robots.txt
 http://app.box.com/robots.txt
 http://apple.co/robots.txt
 http://artnews.com/robots.txt
+http://asahi-net.or.jp/robots.txt
 http://au.effectivemeasure.net/robots.txt
 http://austinmoody.net/robots.txt
 http://b.hatena.ne.jp/robots.txt
@@ -45050,7 +45095,6 @@ http://bbc.com/robots.txt
 http://bbcamerica.com/robots.txt
 http://bbcgoodfood.com/robots.txt
 http://bcg.com/robots.txt
-http://bcp.crwdcntrl.net/robots.txt
 http://beautifulthailand99.blogspot.co.uk/robots.txt
 http://bestdocsnetwork.com/robots.txt
 http://bharat.republicworld.com
@@ -45060,7 +45104,7 @@ http://blackholerecordings.com/robots.txt
 http://bleepingcomputer.com/robots.txt
 http://blog.bufferapp.com/robots.txt
 http://businessbuiltonline.blogspot.com/robots.txt
-http://businesstoday.in/robots.txt
+http://buycatfleamedicine.com/robots.txt
 http://cambridge.org/robots.txt
 http://carinon.hatenablog.com/robots.txt
 http://cdn.rawgit.com/robots.txt
@@ -45074,16 +45118,17 @@ http://comunitaqueeniana.weebly.com/news-of-the-world
 http://comunitaqueeniana.weebly.com/news-of-the-world/category/brian-may
 http://comunitaqueeniana.weebly.com/news-of-the-world/category/freddie-mercury
 http://comunitaqueeniana.weebly.com/news-of-the-world/category/roger-taylor
+http://constantcontact.com/robots.txt
 http://courses.i-leadonline.com/robots.txt
 http://cultofmac.com/robots.txt
-http://d.turn.com/robots.txt
+http://d3js.org/robots.txt
 http://dc.arrivalist.com/robots.txt
 http://delivery.yandex.ru/robots.txt
 http://digitalfx.uk/robots.txt
 http://ecma-international.org/robots.txt
 http://economie.gouv.fr/robots.txt
 http://ed.gov/robots.txt
-http://eff.org/robots.txt
+http://efiles.portlandoregon.gov/robots.txt
 http://en.support.wordpress.com/robots.txt
 http://engadget.com/robots.txt
 http://ericsson.com/robots.txt
@@ -45112,7 +45157,7 @@ http://gyro13.com/robots.txt
 http://hbs.edu/robots.txt
 http://health.com/robots.txt
 http://health.pa.gov/robots.txt
-http://heart.org/robots.txt
+http://hearthis.at/robots.txt
 http://htmlpreview.github.io/robots.txt
 http://hubs.ly/robots.txt
 http://hyperallergic.com/32904/speaking-out-against-commercial-photography-as-art/http://hytaleguide.net/robots.txt
@@ -45130,6 +45175,7 @@ http://istanbulyelken.org.tr/robots.txt
 http://izioq.bandcamp.com/robots.txt
 http://journals.plos.org/robots.txt
 http://jupiterbroadcasting.com/robots.txt
+http://kongregate.com/robots.txt
 http://larkenequity.com/robots.txt
 http://lg.com/robots.txt
 http://librarything.com/robots.txt
@@ -45137,10 +45183,9 @@ http://list.ly/robots.txt
 http://loom.com/robots.txt
 http://m.youtube.com/robots.txt
 http://macobserver.com/robots.txt
-http://mamp.info/robots.txt
+http://managementconsulted.com/robots.txt
 http://market.android.com/robots.txt
 http://mathworld.wolfram.com/robots.txt
-http://memrise.com/robots.txt
 http://militarytimes.com/robots.txt
 http://mineyour.biz/robots.txt
 http://mootools.net/robots.txt
@@ -45941,7 +45986,6 @@ idema
 identification
 identite
 identity
-identity/
 identity/accountsettingsactivityweb
 identity/accountsettingssyaf
 identity/how-we-look
@@ -46146,7 +46190,6 @@ il-en*/people/*/circle
 il-en*/people/*/favorites
 il-en*/shop/*/favoriters
 il-en*/shop/*/sold
-il-en/
 il-en/*?order=
 il-en/*color=
 il-en/*customizable=
@@ -46243,6 +46286,7 @@ image/viewer.htm
 imageClipperUpload
 image_site
 imagelib
+imagelibrary
 imagelightbox?elementId
 imagenes
 imagenes/novias/20201021177551/vestido-novia-desmontable-viral-romancera/*/*.jpg
@@ -46279,6 +46323,8 @@ images/about/
 images/adspace/
 images/api
 images/application_uploads
+images/async/?
+images/async?
 images/campaign/
 images/carBlogsimages
 images/chart_gifs
@@ -46291,6 +46337,9 @@ images/dopfiles/DNT20170204041207.pdf
 images/email/
 images/ga_img_handler.php
 images/home/
+images/hosted/?
+images/hosted/search?
+images/hosted?
 images/interview_pdfs
 images/intfood/thermomix/HF_icon_thermomix.svg
 images/li.gif
@@ -46305,9 +46354,14 @@ images/pad/search?text=*&source=morda
 images/related
 images/resume
 images/scaled
+images/search%3F
+images/search%5C
+images/search/?
 images/search/details/twitterCards/twitter-card
+images/search?
 images/search?rpt=imageview
 images/search?text=*&source=morda
+images/searchbyimage
 images/showroomimages
 images/smart/$
 images/staff
@@ -46323,6 +46377,9 @@ images/verify_image.png
 images/viewconfig$
 images/visit/
 images/welcome_image.png
+images?
+images?*mmbaseline*
+images?*nr*
 imagesLinks
 imagify-backup/
 imagine/
@@ -46413,7 +46470,6 @@ in-en*/people/*/circle
 in-en*/people/*/favorites
 in-en*/shop/*/favoriters
 in-en*/shop/*/sold
-in-en/
 in-en/*?order=
 in-en/*color=
 in-en/*customizable=
@@ -46570,6 +46626,7 @@ inbox-talk
 inbox-user
 inbox.aspx
 inbox.php
+inboxcommerce
 inc
 inc/
 inc5000dev
@@ -48769,6 +48826,7 @@ jkmitchell
 jkn
 jkoivuniemi
 jkwok
+jl
 jladams
 jladapo
 jlc
@@ -49486,7 +49544,6 @@ js2
 js3
 js3/ta-mobile
 js3/vr-shared
-js?
 js?*
 jsErrAjax
 jsGalleryWidget
@@ -50221,6 +50278,7 @@ kontakty
 kopl
 kopt
 koray
+korean
 korina
 korisauserzachrison
 koro
@@ -50250,6 +50308,7 @@ kperdue
 kperkins
 kpi
 kpjmckeon
+kpn
 kporter
 kqin
 kr
@@ -50810,6 +50869,8 @@ learningarcade/
 learningcenter
 learnmore
 learnmore-transferhttp://mc.yandex.ru/robots.txt
+learnmorelayer
+learnmorelayertv
 leave
 leave-a-legacy/legacy-guide/
 leave-a-legacy/thank-you/
@@ -51483,6 +51544,7 @@ loans
 loc/volunteers
 loc=
 local
+local
 local-dev
 local-dev-old
 local-navigation/
@@ -51528,6 +51590,7 @@ locales
 localhistory
 localhost_dump.rar
 localization
+localization-request.php
 localization/
 localize
 localize/
@@ -51660,13 +51723,17 @@ login_or_join/redirected?
 login_or_join?
 login_page
 login_with_redirect
+loginhelp
 loginhelp/
+loginhelp/lookup
+loginhelpsucess
 loginidentification/
 logo
 logo-maker/brief
 logo-maker/choose-variation
 logo-maker/woo
 logo-maker/wordpress
+logo-request.php
 logodownload.php
 logonUserServicesStartupAction.do
 logos
@@ -51681,7 +51748,6 @@ logout.html
 logout.jsp
 logout.php
 logout.pl
-logout/
 logout=
 logout?
 logout?redirectPath=
@@ -52494,6 +52560,7 @@ mails
 mails/
 mailster
 mailto
+mailto.php
 main
 main.php
 main/
@@ -52629,8 +52696,6 @@ manta/sample-pdf
 manteby
 mantis
 manual
-manual/add-note.php
-manual/vote-note.php
 manuelcasas
 manuelhoffmann
 manuelmaqueda
@@ -52673,6 +52738,7 @@ maps/*?lang=uk$
 maps/*?lang=uz$
 maps/*?ll=
 maps/about
+maps/adsendpoint
 maps/api/js
 maps/api/place/js
 maps/api/staticmap
@@ -53034,6 +53100,7 @@ maxboland
 maxcdn.bootstrapcdn.com
 maxencecaillaud
 maximillian-soltysiak
+maxis
 maxkuheljbugaric
 maxnguyen
 maxp=
@@ -53148,6 +53215,7 @@ mcclintock
 mccom-admin/
 mccormack/forms/cameroon/thank-you/
 mccray
+mcd
 mcdowell
 mcgi
 mchan
@@ -53790,7 +53858,6 @@ membership?_ptid=
 membre
 memebers/profile
 memorandum
-memorial/search
 memories?
 mena/errors
 mengchenzheng
@@ -53807,6 +53874,7 @@ menu-test
 menu/*/feedback_form
 menu?
 menumachine
+meo
 meopa/search/node
 meps/*/eptvAjax
 meps/*/pdf*
@@ -53826,6 +53894,7 @@ merchant/directions
 merchant/imageUploadPreview
 merchant/loc
 merchant/moreProducts?
+merchant/reviews?
 merchant/update
 merchant_login
 merchant_shipping_policy.pdf
@@ -54041,6 +54110,7 @@ mietwagen/ergebnisse
 migrate
 migrated
 migration
+migration/
 migration_catalog
 miguelangelsantos
 mihalykalman
@@ -54274,7 +54344,6 @@ misc/social-reader
 misc/tabletapp
 mishu
 missing-email
-missing.html
 missing_link
 missingpage
 mission-focused-transformation
@@ -54700,7 +54769,6 @@ mobility/search
 mobolaji
 mockup-editor
 mod
-mod.php
 mod/forum/search.php
 mod=
 mod?
@@ -54730,6 +54798,7 @@ modern-formats-eng/ucp/static/*/client
 modern-formats-eng/ucp/static/*/csr
 modern-formats-eng/ucp/static/*/dev
 modern-formats-eng/ucp/static/framework_logging
+modernizr
 modes/driving/te-contact-us-make-an-enquiry-failure-26705
 modes/driving/te-contact-us-make-complaint-failure-*
 modify.php
@@ -55527,7 +55596,6 @@ mx*/shop/*/sold
 mx*/step
 mx*/submit
 mx*/submitted
-mx-es/
 mx-es/shop/*?*&*&*&*
 mx-mx/
 mx/
@@ -55656,7 +55724,6 @@ my-bag
 my-bookmarks
 my-books
 my-cart
-my-en/
 my-en/shop/*?*&*&*&*
 my-eurlex
 my-favorites
@@ -57308,6 +57375,7 @@ ninad
 ninahalty
 ninamorales
 ninos/especiales
+niosh/archive/
 nir
 niranjankhadka
 niruban
@@ -58159,7 +58227,6 @@ no-en/your
 no-follow
 no-nb/
 no-nb/docs
-no-no/
 no-route
 no-such-url.html
 no/
@@ -58365,6 +58432,7 @@ not-found
 not-supported.html
 not_found
 not_recommended_reviews
+notamember
 note
 notebook/
 notebook/search?
@@ -58373,6 +58441,7 @@ notfound
 notfound.html
 notice/
 noticeboard
+notices
 noticias/20080416/destituyen-director-jefe-proteccion-asco-tras-fuga-radiactiva/28170.shtml
 noticias/envia_noticia.html
 noticias/live-fyre-blog-test*
@@ -58385,6 +58454,7 @@ notificationId
 notifications
 notifications
 notifications$
+notifications/
 notifications/*
 notify.js
 notifyMe.jsp
@@ -58439,6 +58509,7 @@ npr
 nqrwns
 nr/
 nraphael
+nrdapp
 nreed
 nrg
 nripsuta
@@ -58552,7 +58623,6 @@ nz*/shop/*/sold
 nz*/step
 nz*/submit
 nz*/submitted
-nz-en/
 nz-en/shop/*?*&*&*&*
 nz/
 nz/*
@@ -58850,6 +58920,8 @@ offers/nonprofits$
 offers/nonprofits%
 offers/nonprofits?
 offers/preview/
+offers/proxy/dealsserver/api/log
+offers/proxy/dealsserver/buy
 offers/referral-terms-conditions
 offers/referral-terms-conditions$
 offers/referral-terms-conditions%
@@ -58998,7 +59070,7 @@ omniauth
 omniture/
 on
 on
-on-demand/session/*http://nxp.com/robots.txt
+on-demand/session/*http://news-medical.net/robots.txt
 on/*.ico$
 on/*.png$
 on/demandware.store
@@ -59213,6 +59285,7 @@ or/user
 or/users
 oral-and-maxillofacial-surgeon
 orange-anma
+orangefrance
 orcid-search
 order
 order
@@ -59389,6 +59462,7 @@ override
 override/
 overview
 overview.php
+overviewblockseeother
 owasow
 owenmartin
 owner-estimate
@@ -60026,6 +60100,7 @@ pamr/issues_experts/featured_experts_archive/index.html
 pamr/issues_experts/issues_experts_archive/index.html
 pamr/media_releases/index.html
 pamr/media_releases/media_releases_archive/index.html
+panasonic
 panda/jira
 panel
 panel
@@ -60580,7 +60655,6 @@ ph*/cdn-cgi/challenge-platform
 ph*/step
 ph*/submit
 ph*/submitted
-ph-en/
 ph/
 ph/*?*shpxid=
 ph/*?services
@@ -60762,6 +60836,7 @@ philippahetherington
 phillai
 phillies/fans/text-messaging-terms-and-conditions
 phillies/search
+phishwatch@lists.clean-mx.com
 phl-avvo-home
 phl-migration
 phoenix/api
@@ -60770,6 +60845,7 @@ phone-fax-purchases.html
 phone/compare/?
 phonebook
 phoneinfo
+phonenumber
 phot-browse.asp*
 photo
 photo-
@@ -60904,6 +60980,7 @@ pift/retest
 pigeon
 pilot-rijbewijs
 pimages
+pin
 pin-builder
 pin-communities
 pin-creation-tool
@@ -61395,6 +61472,7 @@ playlist.xml
 playlists/*/user
 playlists/flag
 plays
+playstation
 plcz
 plde
 plehmann
@@ -61893,6 +61971,7 @@ populate
 populate/
 popup
 popup.php
+popup/codewhatisthis
 popup/emailconfirmation.xhtml
 popup=
 popup=true
@@ -61900,6 +61979,8 @@ popup?
 popup_salesforce.html
 popup_security.jsp
 popupclarin.asp
+popupdetails
+popupprivacypolicy
 popups/
 popups/mediaplayer
 popups/popup_gallery
@@ -62392,6 +62473,7 @@ privacy/request-data
 privacy/zoho-group.html
 privacy_policy
 privacypolicy/
+privacypolicychanges
 private
 private
 private-promo
@@ -62441,6 +62523,7 @@ pro/tools
 pro/upgrade
 pro/waze/report/*?*&hash=
 pro2
+proFile/history?
 proMatch
 proProjects
 proTeXt
@@ -62523,6 +62606,7 @@ product/prd-5193422
 product/productreviews/
 product/recommended
 product/review/add
+product/reviews?
 product/summer-plant-based-recipe-cookbook
 product/valentines-chocolate-vegan-cookbook
 product/vegan-desserts-cookbook
@@ -62745,6 +62829,7 @@ profile/contributor/?
 profile/downloads_limit
 profile/edit
 profile/gravlady
+profile/history?
 profile/license
 profile/manage
 profile/messages
@@ -63052,6 +63137,7 @@ provi/check
 providers
 provost/articulation
 proximic
+proximus
 proxy
 proxy
 proxy/accounts
@@ -63786,6 +63872,7 @@ q_
 qa
 qa/
 qadministrator/mac/4.0/en/appleqadministrator/usermanual
+qantas
 qantas-group/delivering-today/fy18-highlights.html
 qbw
 qc_campaign
@@ -63904,6 +63991,7 @@ queue
 queue
 queuePartial
 qui-sommes-nous
+qui_sommes_nous
 quick-compare
 quick-reads/*.html
 quick-share/
@@ -64135,7 +64223,6 @@ raven-hubspot/static-1.73
 raven/static-3.4
 raven/static-3.6
 raw
-raw/
 rawson
 ray
 raymond-cs-wong
@@ -64658,6 +64745,7 @@ relocator
 relpage
 remartin
 remember-the-titans
+rememberme
 remingtonhill
 remixed/rules/
 remo
@@ -64692,6 +64780,11 @@ rennekamp
 renos
 renovation
 renta-de-autos/resultados
+renta/sc/frm/coupon
+renta/sc/frm/mypage
+renta/sc/frm/rent
+renta/sc/frm/report
+renta/sc/read_free.cgi
 rentacar/byr/
 rentacar/car-hire/
 rentacar/customersupport/index.jsp?targetPage=faqsView.jsp
@@ -66294,6 +66387,7 @@ rewards/$
 rewards/*/membership
 rewards/*/status-matching
 rewards/bebas-ongkir
+rewardsapp/
 rex/recommendation/*
 rezepte/suche
 rf=*
@@ -66692,7 +66786,6 @@ routen/*/routen-ab-
 routen/*/routen-von-
 router
 router/*
-routes.json
 routes/*/cac-duong-bay-tu-
 routes/*/export
 routes/*/routes-from-
@@ -66727,7 +66820,6 @@ rrasband
 rroatman
 rrx219
 rs
-rs-sr/
 rs/businessclub/novo-tc.html
 rs=
 rsamuels
@@ -67334,6 +67426,7 @@ samplescholar
 sampson
 samr
 samrosenfeld
+samsung
 samsus
 samsweney
 samuel_s_urlacher
@@ -67900,7 +67993,6 @@ se-en/your
 se-fi/
 se-rank
 se-se/
-se-sv/
 se-sv/*
 se/
 se/*?*shpxid=
@@ -68692,7 +68784,6 @@ serpilfox
 serv/
 server
 server-configurator
-server-status
 server.php
 server/banners
 server/body-include
@@ -68828,7 +68919,6 @@ services/developmentalumni/giving/chancellors_apply.html
 services/education/sa/
 services/education/saprogram/
 services/emailFriend
-services/engine/search/sru?
 services/estates/2000
 services/finance/Local
 services/finance/agresso
@@ -69062,7 +69152,6 @@ sg-en*/people/*/circle
 sg-en*/people/*/favorites
 sg-en*/shop/*/favoriters
 sg-en*/shop/*/sold
-sg-en/
 sg-en/*?order=
 sg-en/*color=
 sg-en/*customizable=
@@ -69277,6 +69366,7 @@ shelleyxliu
 shellysimana
 shen
 sheng
+shenghuo
 shenghuo/search?
 shenyang
 sheraz
@@ -69322,6 +69412,7 @@ shoelaces
 shomiller
 shootouts
 shop
+shop$
 shop-all
 shop-data
 shop-interrupt
@@ -69369,6 +69460,7 @@ shop/browse/overlay
 shop/browsing_history
 shop/checkout
 shop/claimcredit
+shop/collections
 shop/edit
 shop/eligible-for-special-financing/*?
 shop/flattering-fitted-maxi-dress-5152169
@@ -69407,6 +69499,7 @@ shop/unified/availability/calltoorder.html
 shop/unified/availability/results.html
 shop/view/product
 shop/wishlist
+shop?
 shop_name_search_service
 shop_pins
 shopby
@@ -69544,7 +69637,6 @@ shuxi_qiao
 shyamkakula
 si
 si-LK
-si-en/
 si-lk/
 si/
 si/Download
@@ -69669,6 +69761,7 @@ signup_modal
 signup_twitter.php
 signup_verify.php
 signups
+signurl
 sigou-interdisciplinary-blog
 sijang
 sijumonkunjachan
@@ -69719,6 +69812,7 @@ simpleshare
 simpson
 simpsons2000
 simsek
+simulateur-nouvelle-actualisation
 simwidgets
 sina-kianersi
 sina-sharifi
@@ -70591,7 +70685,6 @@ snp
 sns-ap
 sns/*
 so/
-soa/
 soajax/Eta$
 soap/
 soaplab
@@ -70618,6 +70711,7 @@ social/
 social/our_partners/
 social/photosui/images
 social/photosui/sitemap
+social/search?
 social/share_modal
 social/social-security-directory/cai
 social_media
@@ -70774,6 +70868,7 @@ soniapujol
 sons-mp3-us
 sonumehta
 sonusubudhi
+sony
 soojin
 soojinpark
 sophia_lee
@@ -70860,6 +70955,7 @@ sparshasaha
 spartan
 spaul
 spb
+spbasic
 spbot
 spc-source=
 spc/amazon/en/landingPage.html
@@ -71035,6 +71131,7 @@ sports_fp_ms
 spotlight
 spp/
 spreadsheet
+spresults
 sprillaman
 sprince
 spring
@@ -71209,7 +71306,6 @@ stafftravel/
 stage
 stage-homepage-backup.html
 stage-ranking
-staged-content/
 staging
 stalker
 stamp
@@ -71678,7 +71774,9 @@ storeclick
 storefront
 storepicker
 storerequest
+stores$
 stores-manage
+stores/
 stores/page/*visitId=
 stories
 stories/shop
@@ -71988,6 +72086,7 @@ subscription/*?*source=
 subscription/subscribe
 subscription/subscribe$
 subscriptionDetails.htm
+subscriptioncancel
 subscriptions
 subscriptions/manage/
 subscriptions/securejson/
@@ -73027,6 +73126,7 @@ talk_to_us/?
 talkread
 talks2
 talkscreen.bml
+talktalk
 tallarita
 tally-loader-react/static-1.65
 tallyman/context
@@ -73067,6 +73167,7 @@ tasawarjalali
 tasima
 tasks
 tasks/
+tastesurvey
 tax-center
 tax.php
 tax_countries.php
@@ -73216,6 +73317,7 @@ td/profiles
 tdangelo
 tdaroundme
 tdautourdemoi
+tdc
 tdevine
 tdichter
 tdid=*
@@ -73367,6 +73469,7 @@ teleconnections/*.php?
 telegram998com
 telegraph
 telehealth-for-women
+telenor
 television/entretenimiento
 television/teletexto/incibe
 television/teletexto/noticias
@@ -73377,6 +73480,7 @@ television/tve-internacional/asia/?
 television/tve-internacional/europa/?
 television/tve-internacional/parrilla/?
 television/tve-internacional/programacion-tv/?
+telfort
 tell-friend
 tellAFriend.do
 telsearch
@@ -73486,6 +73590,17 @@ template?
 templateabuse
 templatedata
 templatedata
+templatepackage/3.0/Templates
+templatepackage/3.0/examples
+templatepackage/3.0/templates
+templatepackage/4.0/docs
+templatepackage/4.0/gallery-internal
+templatepackage/Templates
+templatepackage/examples
+templatepackage/gadgets
+templatepackage/modules
+templatepackage/subtopic
+templatepackage/templates
 templates
 templates$
 templates/
@@ -73566,6 +73681,7 @@ terms/looker/legal/suppliers/partner-assignment-faq
 terms/looker/legal/suppliers/supplier-code-of-conduct
 terms_of_use
 termsandprivacy.aspx
+termsofusechanges
 tervala
 tessibrahimchah
 tesswallace
@@ -73708,7 +73824,6 @@ textbook/isbn
 textbook/top-selling-textbooks
 textbooks
 textbooks/_ajax/log
-texteBrut
 textimage
 textonly
 textpattern
@@ -73723,6 +73838,7 @@ tge
 tgibson
 tgkkk6681
 th
+th$
 th*/blog-article-remove-faq-utms-*.js
 th*/cdn-cgi/challenge-platform
 th*/step
@@ -73875,6 +73991,7 @@ th/user
 th/users
 th/ventureone
 th/ส่วนประกอบ_widget
+th?
 th_
 th_th/templates/M
 thailandgolf.
@@ -74246,6 +74363,7 @@ tilda/scroll
 till
 tilleczek
 tilton
+tim
 tim-miller
 tim/index.xhtml
 timanderson
@@ -75050,6 +75168,7 @@ translation
 translation-not-found
 translations
 translations/
+translator/?ref=
 translators
 transmute/static-1.85
 transmute/static-2.6
@@ -75113,10 +75232,18 @@ travel/airlines/main/au/en
 travel/airlines/main/us/en
 travel/airlines/pfs*
 travel/airlines/premium-travel-arranger/au/en
+travel/api
 travel/booking/en_gb#/*
 travel/clk
+travel/css
 travel/entity
 travel/event/search
+travel/flight-search
+travel/flight/flightSearch
+travel/flight/flightSearchAction
+travel/flight/search/?
+travel/flight/search?
+travel/flights
 travel/flights/s
 travel/globalgateway.jsp/global/public/de_
 travel/globalgateway.jsp/global/public/en_
@@ -75145,15 +75272,24 @@ travel/home/public/pt_*
 travel/home/public/ru_*
 travel/home/public/sv_*
 travel/home/public/zh_*
+travel/hotel-search
 travel/hotel/e
+travel/hotel/hotelMiniSearchRequest
+travel/hotel/hotelSearch
+travel/hotels/checkout
+travel/hotels/detail
 travel/hotels/entity
 travel/hotels/hotel-finder
 travel/hotels/hotel-finder/$
+travel/hotels/search/?
+travel/hotels/search?
 travel/hotels/stories
 travel/passport
 travel/pkomatch/
 travel/s
+travel/scripts
 travel/search
+travel/secure
 travel/test$
 travel/wp-admin
 travelDate=
@@ -75405,6 +75541,7 @@ tto
 ttobin
 ttong
 tucritica
+tudum
 tue
 tufengen
 tune*retpath=
@@ -75565,6 +75702,7 @@ tvmag
 tvopham
 tvschedule
 tvschedules
+tvsignup
 tvtee
 tw
 tw*/blog-article-remove-faq-utms-*.js
@@ -75647,7 +75785,6 @@ twitter
 twitter/ptt.php
 twitter_impression*
 twittercard
-tx_form_formframework
 tx_solr*
 txhan
 txt
@@ -75668,7 +75805,6 @@ typo3/
 typo3/sysext/frontend/Resources/Public/*
 typo3_src/
 typo3conf
-typo3conf/ext
 typo3conf/ext/parallels/Resources/Public
 typo3conf/ext/prls_theme/Resources/Public
 typo3temp
@@ -76132,9 +76268,11 @@ unijobs/previewjob
 unijobs/searchjobs
 unijobs/session-img
 unijobs/your-jobs
+unijuridis
 uninstall
 union.php
 union/planner
+unipublics
 uniqifyingToken=
 unit
 unit-programme-catalogue
@@ -76221,6 +76359,7 @@ until=
 up
 upcoming-parties
 upcomingeventdetails.html
+upcomingevents
 upcomingevents.html?
 update
 update-check/wirecast/
@@ -76317,6 +76456,7 @@ ur/users
 urayama
 urchin_test
 url
+url
 url-shortner
 url.asp
 url=
@@ -77091,7 +77231,7 @@ utl/
 utm
 utm
 utm*=
-utm*http://funbaby.cz/robots.txt
+utm*http://fun88thai.net/robots.txt
 utm=
 utm=
 utm_
@@ -77346,6 +77486,7 @@ verify_age
 verify_cookies
 verify_your_email$
 verify_your_email?
+verifyidentity
 verizon
 verizon-connecta-la-banda*
 veronica_benavides
@@ -77676,6 +77817,7 @@ videoLibrary/assets/swf
 videoLibrary/common/services
 videoLibrary/transcript
 videoLogin?
+video_
 video_file
 video_test_page
 videocard
@@ -77693,28 +77835,45 @@ videorequest
 videos
 videos/*.feedback.html
 videos/*?
+videos/?
 videos/api
 videos/assets
+videos/browsing
 videos/e
 videos/expander
+videos/explore
+videos/favorites
+videos/feed
+videos/feedcontent
 videos/get_video_by_uid/
 videos/hardware-in-the-loop-simulation-with-simulink-real-time-using-plcs-and-ethernetip-protocol-for-validating-industrial-automation-control-algorithms-96697.html
 videos/iframe
 videos/images
 videos/js
+videos/music
+videos/music/
+videos/music/?
+videos/music?
 videos/new
 videos/raw-ethernet-communication-support-from-simulink-real-time-96619.html
 videos/real-time-udp-communication-support-99895.html
+videos/riverview/relatedvideo
+videos/riverview/relatedvideo/?
+videos/riverview/relatedvideo?
 videos/search
 videos/search.html
 videos/search.php
+videos/search/?
 videos/search?
 videos/show_trailer
 videos/slider
 videos/transcripts
+videos/trending
+videos/trendingcontent
 videos/video-feedback-confirmation.html
 videos/video-feedback.html
 videos/viewmeta/
+videos?
 videosearch
 videosearch?
 videotutorials
@@ -77762,6 +77921,7 @@ viewconfig$
 viewer
 viewfile.php
 viewfound.php
+viewingactivity
 viewlocale=zh_CN
 viewonline.php
 vieworder
@@ -77814,6 +77974,7 @@ vionnieyu
 vip
 vip_club
 viral-picks
+virginaustralia
 virginialovison
 virtual
 virtual-open-day
@@ -77885,7 +78046,6 @@ vn*/cdn-cgi/challenge-platform
 vn*/step
 vn*/submit
 vn*/submitted
-vn-vi/
 vn/
 vn/*?*shpxid=
 vn/*digital_wallets/dialog
@@ -77946,7 +78106,9 @@ vocabolario/ricerca/*量
 vocabolario/tag
 vocabularies
 vod
+vodafone
 vodafone-foundation/focus-areas/dreamlab-app/Lesotho
+vodafonedemobilelaunch
 voguevoices/thankyou
 voguevoices/thankyou2
 voiceover
@@ -83076,7 +83238,6 @@ web/packages/ROptEst/DESCRIPTION
 web/packages/ROptSpace/DESCRIPTION
 web/packages/ROptimus/DESCRIPTION
 web/packages/ROracle/DESCRIPTION
-web/packages/RPANDA/DESCRIPTION
 web/packages/RPDTest/DESCRIPTION
 web/packages/RPEGLMEN/DESCRIPTION
 web/packages/RPEIF/DESCRIPTION
@@ -94080,7 +94241,6 @@ web/packages/oro.nifti/DESCRIPTION
 web/packages/oro.pet/DESCRIPTION
 web/packages/orsifronts/DESCRIPTION
 web/packages/orsk/DESCRIPTION
-web/packages/orthGS/DESCRIPTION
 web/packages/orthoDr/DESCRIPTION
 web/packages/orthogonalsplinebasis/DESCRIPTION
 web/packages/orthopolynom/DESCRIPTION
@@ -100491,14 +100651,19 @@ why-does-digital-privacy-matter-i-have-nothing-to-hide
 why-ghostery-midnight
 why-liveperson
 why-sell?inquirySource=482&
+whysecure
 whythisad
 wicket:interface*http://springerlink.com/robots.txt
 widget
 widget-landing-page.html
 widget-uploads
 widget.html
+widget/cr
+widget/entity/search/?
 widget/head/login
 widget/player
+widget/render
+widget/snapshot
 widget/v2
 widget_auth.php
 widget_chunks
@@ -101636,6 +101801,7 @@ wordpress/wp-admin
 wordpress/wp-admin/admin-ajax.php
 words
 work
+work
 work$
 work-at-ucl/redeployment
 work-info
@@ -101895,6 +102061,7 @@ wp-admin/admin-ajax.phphttp://aljazeera.com/robots.txt
 wp-admin/admin-ajax.phphttp://ask.com/robots.txt
 wp-admin/admin-ajax.phphttp://aweber.com/robots.txt
 wp-admin/admin-ajax.phphttp://blogs.lse.ac.uk/robots.txt
+wp-admin/admin-ajax.phphttp://derstandard.at/robots.txt
 wp-admin/admin-ajax.phphttp://findbyplate.com/robots.txt
 wp-admin/admin-ajax.phphttp://foodandwine.com/robots.txt
 wp-admin/admin-ajax.phphttp://goldensun.get.com.tw/robots.txt
@@ -101902,11 +102069,9 @@ wp-admin/admin-ajax.phphttp://hopemoon.com/robots.txt
 wp-admin/admin-ajax.phphttp://johnallman.uk/robots.txt
 wp-admin/admin-ajax.phphttp://law.cornell.edu/robots.txt
 wp-admin/admin-ajax.phphttp://publish.get.com.tw/robots.txt
-wp-admin/admin-ajax.phphttp://royalfurniture.com/robots.txt
 wp-admin/admin-ajax.phphttp://skaterki.ru/robots.txt
 wp-admin/admin-ajax.phphttp://smugmug.com/robots.txt
 wp-admin/admin-ajax.phphttp://toucharcade.com/robots.txt
-wp-admin/admin-ajax.phphttp://vox.com/robots.txt
 wp-admin/admin-ajax.phphttp://wera-heilbronn.de/robots.txt
 wp-admin/admin-ajax.phphttp://zenodo.org/robots.txt
 wp-admin/admin-ajax.phphttp://zoho.com/robots.txt
@@ -102107,7 +102272,6 @@ wp-content/uploads/wp-import-export-lite/http://amnh.org/robots.txt
 wp-content/uploads/wp-import-export-lite/http://bigthink.com/robots.txt
 wp-content/uploads/wp-import-export-lite/http://realmsoftrinity.com/robots.txt
 wp-content/uploads/wp-import-export-lite/http://susanheat.de/robots.txt
-wp-content/uploads/wp-import-export-lite/http://theridesharepro.com/robots.txt
 wp-content/uploads/wp-import-export-lite/http://youblisher.com/robots.txt
 wp-content/uploads/wpforms
 wp-content/uploads/wpforms/
@@ -102350,6 +102514,7 @@ x?
 xArchive/Archive/Illegal-liquor-export-20010319
 xavier
 xb/en/
+xboxone
 xbp/search
 xc/
 xclick-auction
@@ -102365,6 +102530,7 @@ xd/health/services/family-medicine/clinic/nike-employee.cfm
 xdong
 xdoor
 xfilter=1
+xfinity
 xfx7
 xh/forums
 xh/kb/*/discuss
@@ -102446,6 +102612,7 @@ xorouter
 xpert/xcloud
 xqli
 xs
+xs4all
 xsanR1
 xsanR2
 xsanR3
@@ -102618,6 +102785,8 @@ your-library-invoices.asp*
 your-lyst
 your-orders
 your-shop
+yourAccount
+youraccount
 youraccount/
 yourarchives
 youreurope/businessACC
@@ -102729,7 +102898,6 @@ za*/cdn-cgi/challenge-platform
 za*/step
 za*/submit
 za*/submitted
-za-en/
 za/
 za/*?*shpxid=
 za/*?services
@@ -103280,6 +103448,7 @@ ziad
 zibell
 zichenhms
 zickler
+ziggo
 zihaowang
 zillowarticles
 zimmermann
@@ -103912,10 +104081,12 @@ zzz_systest
 7176/
 7176/TOI_Travel/
 7176/TOI_Travel_Test/TOI_Travel_TOP_TST
+7336
 7449
 7577
 8058
 8245
+8264
 8324
 8601
 8614