[Github Action] Automated trickest wordlists update.

.bin/wordlist-updaters/status.json

@@ -3,6 +3,6 @@
         "last_update": 1724007797
     },
     "Trickest wordlist update": {
-        "last_update": 1723975420
+        "last_update": 1724061833
     }
 }

Discovery/Web-Content/CMS/trickest-cms-wordlist/directus-all-levels.txt

@@ -16526,6 +16526,8 @@ license
 contributing.md
 .changeset/config.json
 config.json
+.changeset/quick-worms-kick.md
+quick-worms-kick.md
 .changeset/wise-bikes-suffer.md
 wise-bikes-suffer.md
 .changeset/gentle-balloons-pretend.md

Discovery/Web-Content/CMS/trickest-cms-wordlist/directus.txt

@@ -3345,6 +3345,7 @@ app/src/lang/get-current-language.test.ts
 app/license
 contributing.md
 .changeset/config.json
+.changeset/quick-worms-kick.md
 .changeset/wise-bikes-suffer.md
 .changeset/gentle-balloons-pretend.md
 contributors.yml

Discovery/Web-Content/CMS/trickest-cms-wordlist/dolibarr-all-levels.txt

@@ -7057,6 +7057,10 @@ propal_extrafields.php
 htdocs/comm/recap-client.php
 comm/recap-client.php
 recap-client.php
+htdocs/intracommreport/lib/intracommreport.lib.php
+intracommreport/lib/intracommreport.lib.php
+lib/intracommreport.lib.php
+intracommreport.lib.php
 htdocs/intracommreport/list.php
 intracommreport/list.php
 list.php
@@ -64900,10 +64904,6 @@ htdocs/core/lib/payments.lib.php
 core/lib/payments.lib.php
 lib/payments.lib.php
 payments.lib.php
-htdocs/core/lib/intracommreport.lib.php
-core/lib/intracommreport.lib.php
-lib/intracommreport.lib.php
-intracommreport.lib.php
 htdocs/core/lib/contact.lib.php
 core/lib/contact.lib.php
 lib/contact.lib.php

Discovery/Web-Content/CMS/trickest-cms-wordlist/dolibarr.txt

@@ -1480,6 +1480,7 @@ htdocs/comm/propal/agenda.php
 htdocs/comm/admin/propaldet_extrafields.php
 htdocs/comm/admin/propal_extrafields.php
 htdocs/comm/recap-client.php
+htdocs/intracommreport/lib/intracommreport.lib.php
 htdocs/intracommreport/list.php
 htdocs/intracommreport/card.php
 htdocs/intracommreport/admin/intracommreport.php
@@ -13082,7 +13083,6 @@ htdocs/core/actions_massactions.inc.php
 htdocs/core/actions_extrafields.inc.php
 htdocs/core/lib/socialnetwork.lib.php
 htdocs/core/lib/payments.lib.php
-htdocs/core/lib/intracommreport.lib.php
 htdocs/core/lib/contact.lib.php
 htdocs/core/lib/profid.lib.php
 htdocs/core/lib/supplier_proposal.lib.php

Discovery/Web-Content/CMS/trickest-cms-wordlist/opencart-all-levels.txt

@@ -33739,11 +33739,6 @@ system/library/session/redis.php
 library/session/redis.php
 session/redis.php
 redis.php
-upload/system/library/session/mock.php
-system/library/session/mock.php
-library/session/mock.php
-session/mock.php
-mock.php
 upload/system/library/session/file.php
 system/library/session/file.php
 library/session/file.php

Discovery/Web-Content/CMS/trickest-cms-wordlist/opencart.txt

@@ -3492,7 +3492,6 @@ upload/system/library/cache.php
 upload/system/library/template/twig.php
 upload/system/library/template/template.php
 upload/system/library/session/redis.php
-upload/system/library/session/mock.php
 upload/system/library/session/file.php
 upload/system/library/session/db.php
 upload/system/library/request.php

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt

@@ -1,7 +1,6 @@
 $
 $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 %24rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
-&solved=
 ,
 -
 -/en$
@@ -46,14 +45,12 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /.git
 //wiki/skins
 /MANUALS/*.pdf$
-/Search/
 /_
 /_/rsrc
 /album/*/*?i=
 /api
 /apis
 /archive
-/base-en/
 /bin/search.py
 /blame
 /cache
@@ -62,14 +59,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /commits/*?path
 /docs/product
 /download
-/download/Browse.aspx?
-/download/browse.aspx?
-/download/confirmation.aspx?
-/download/info.aspx?
-/download/registration-suggested.aspx?
-/download/results.aspx?
-/download/thankyou.aspx
-/download/thankyou.aspx?
 /download?
 /edit$
 /favorites_list
@@ -81,16 +70,11 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /format:thumbnail
 /forum-attachment
 /gcrnsts
-/layout/
 /leads/*/jsplayer_show_lead_form.html
 /leads/*/show_form.html
 /listing/*/favoriters
 /lookup?
-/mediaapi/
 /mediaviewer/*/tr
-/music/*/Search/
-/music/*/search/
-/newsearch/
 /people/*/circle
 /people/*/favorites
 /pinned_flashes
@@ -101,26 +85,15 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /rg*/mediaviewer/rm*/tr
 /rss
 /search
-/search/
 /search/*?q=
 /search?q=
-/security/search-results.aspx?
 /shop/*/favoriters
 /shop/*/sold
 /stats?id=
-/store/*/layout/
-/store/*/search/
-/store/buynow?
-/store/cart
-/store/d/groove-music-pass/cfq7ttc0k5dq/0001
-/store/locations/store-locations-sitemap.xml
-/store/music/groove-music-pass/*
 /tarball
 /tree
 /tv-season/*/*?i=
-/unsubscribe/
 /video-
-/wal/
 /wiki/Special:
 /wiki/index.php/Help
 /wiki/index.php/MediaWiki
@@ -133,7 +106,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /wp-content/themes
 /wp-includes
 /zipball
-=imgmanager
 ?
 ?*&ls=
 ?*ListingID=
@@ -185,7 +157,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?hl=*&
 ?hl=*&*&gws_rd=ssl
 ?hl=*&gws_rd=ssl$
-?intc=
 ?lang=
 ?ls=
 ?null$
@@ -219,7 +190,6 @@ AccountMerge
 Achievements
 ActLikeAnOwner
 ActionRecord
-ActivityUID=
 AdRequestEventLogApi
 AddForumUser
 AddListing
@@ -385,8 +355,6 @@ GSSAjax
 GeneralSupport
 GermanwingsPromo
 GermanwingsRegistration
-Germany/kleinunternehmen/euga/detail.mspx?
-Germany/kleinunternehmen/euga/results.mspx?
 GetListedAjaxGeneral
 GetListedLightbox
 GetListedUpdateAttraction
@@ -423,7 +391,6 @@ HotelsManageListing
 HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
-HpOptOut=true$
 IBCommerce
 ImpressionRecord
 ImproveListing
@@ -441,7 +408,6 @@ International/questions/qa-html-language-declarations/icons
 InterstitialsWidget
 IpadInterstitial
 JSErrorRecord
-Japan/DirectX/default.asp?
 JoinPlus
 JsonServiceController
 JustForYouGuide
@@ -863,7 +829,6 @@ TAScreenSaver
 TCDestinationsWidget
 TCIBContentQualityTool
 TLMCountries
-TOCLinksForCrawlers*
 Tablet
 TaggingAjax
 TagsJSONServlet
@@ -1111,38 +1076,6 @@ accounts/force-by-email
 accounts/o8
 accounts/o8/id
 aclk
-action=accessorysearch&
-action=accessorysearch&page=2$
-action=accessorysearch&page=3$
-action=accessorysearch&page=4$
-action=accessorysearch&page=5$
-action=accessorysearch&page=6$
-action=accessorysearch&page=7$
-action=accessorysearch&page=8$
-action=accessorysearch&product=*$
-action=accessorysearch&product=*&*
-action=catalogsearch&
-action=catalogsearch&catalog_mode=grid&page=2$
-action=catalogsearch&catalog_mode=grid&page=3$
-action=catalogsearch&catalog_mode=grid&page=4$
-action=catalogsearch&catalog_mode=grid&page=5$
-action=catalogsearch&catalog_mode=grid&page=6$
-action=catalogsearch&catalog_mode=grid&page=7$
-action=catalogsearch&catalog_mode=grid&page=8$
-action=catalogsearch&catalog_mode=list&page=2$
-action=catalogsearch&catalog_mode=list&page=3$
-action=catalogsearch&catalog_mode=list&page=4$
-action=catalogsearch&catalog_mode=list&page=5$
-action=catalogsearch&catalog_mode=list&page=6$
-action=catalogsearch&catalog_mode=list&page=7$
-action=catalogsearch&catalog_mode=list&page=8$
-action=overlay&
-action=productLinkAction&
-action=productSupportSearch
-action=productcompareaction&
-action=quickSearch&
-action=siteSearch
-action=writeReview
 actions/news/rss
 ad_acknowledgment
 ad_id=
@@ -1483,7 +1416,6 @@ biz_redir
 biz_share
 biz_update
 bkshp?*q=
-blacklisted*
 blog/*/*/comments
 blog/*/*/feed
 blog/*/*/trackback
@@ -1649,7 +1581,6 @@ calendar/about
 call/join
 call?id=
 calls
-canada/Library/mnp/2/aspx/
 cap
 card/panel
 cart
@@ -1773,10 +1704,6 @@ commercesearch/images
 commercesearch/services
 commission_2010-2014/katainen
 commits?author=
-communities/bin.aspx?
-communities/blogs/PortalResults.mspx?
-communities/eventdetails.mspx?
-communities/rss.aspx*
 communities/sitemap
 community
 community/*/category/*
@@ -1840,7 +1767,6 @@ de*/people/*/favorites
 de*/shop/*/favoriters
 de*/shop/*/sold
 de-de.www.cloudflare.com
-de-de/corporate/rechtliche-hinweise/impressum_de.aspx
 de-de/feedback
 de-de/lp
 de-en*/listing/*/favoriters
@@ -2139,15 +2065,6 @@ en-in.www.cloudflare.com
 en-in/feedback
 en-in/lp
 en-us.www.cloudflare.com
-en-us/windows/si/matrix.html
-en/microsoft-365
-en/microsoft-365/
-en/microsoft-teams
-en/microsoft-teams/
-en/microsoft-viva
-en/microsoft-viva/
-en/security
-en/security/
 endorsements
 enterprise-jobs
 enterprise-partner-search/sitemaps
@@ -2196,20 +2113,12 @@ es/market/confederate_flag
 es/market/dixie_flag
 es/market/rebel_flag
 es/messages
-es/microsoft-365
-es/microsoft-365/
-es/microsoft-teams
-es/microsoft-teams/
-es/microsoft-viva
-es/microsoft-viva/
 es/mini.php
 es/people
 es/search*attr_*=
 es/search*price_bucket=
 es/search*search_type=
 es/search*ship_to=
-es/security
-es/security/
 es/share
 es/shop_name_search_service
 es/teams
@@ -2260,7 +2169,6 @@ facebook
 facebook_connect
 faves-i_love_u_get_away_from_me
 favorite
-fe=true
 feature/sf-test
 feed
 feed$
@@ -2271,7 +2179,6 @@ feed?section=stayhome
 feed?section=stayhome$
 feedback
 feeds
-feeds/TechNet/fr-fr/screenshot/screenshot%20surface.jpg
 feeds/videos.xml
 fetch
 fi-en*/listing/*/favoriters
@@ -2392,29 +2299,18 @@ fr/market/confederate_flag
 fr/market/dixie_flag
 fr/market/rebel_flag
 fr/messages
-fr/microsoft-365
-fr/microsoft-365/
-fr/microsoft-teams
-fr/microsoft-teams/
-fr/microsoft-viva
-fr/microsoft-viva/
 fr/mini.php
 fr/people
 fr/search*attr_*=
 fr/search*price_bucket=
 fr/search*search_type=
 fr/search*ship_to=
-fr/security
-fr/security/
 fr/share
 fr/shop_name_search_service
 fr/teams
 fr/transactions
 fr/treasury
 fr/your
-france/formation/centres/planning.asp?
-france/ie/default.asp?
-france/mnp_utility.mspx?
 friends
 from_id=
 frommers
@@ -2430,9 +2326,6 @@ gamtarget
 gb/images
 gcc
 gdpr-consent
-genuine/
-germany/library/images/mnp/
-germany/video/de/de/related*
 getCallUsInfoData
 get_midroll_info
 get_video
@@ -2641,9 +2534,8 @@ hosted/images
 hotelfinder/rpc
 hotels/rpc
 hp/video
-hpc/*/supported-applications.aspx?
-http://amzn.to/robots.txt
-http://dropbox.com/robots.txt
+http://archive.org/robots.txt
+http://cloudflare.com/robots.txt
 hub?category=
 hz/contact-us/ajax/initiate-trusted-contact
 hz/help/contact/*/message/$
@@ -2684,7 +2576,6 @@ ie/conversations
 ie/guest-pass
 ie/help/sitemap
 ie/help/sitemap.php
-ie/ie40/
 ie/invites/buy
 ie/listing/*/stubs
 ie/locale/overlay
@@ -2761,7 +2652,6 @@ il-en/your
 il/cart
 im?
 images
-imaginecup/*
 imesync
 imgres
 imode/?
@@ -2823,13 +2713,10 @@ includes
 indepthtoolkit/smallprox
 indepthtoolkit/smallprox$
 index.html?
-info/customerror.htm*
-info/smart404.asp*
 info/tech-feedback
 inproduct_help/chatsupport
 internalPref
 intl/*/about/views
-intlkb/
 inviteFromProfile
 inviteMany
 invites/buy
@@ -2848,7 +2735,6 @@ iplayer/cbeebies/search
 iplayer/search
 ireceipt/get
 ireceipt/get?
-isapi/
 it*/listing/*/favoriters
 it*/people/*/circle
 it*/people/*/favorites
@@ -2905,11 +2791,6 @@ it/your
 ja-jp.www.cloudflare.com
 ja-jp/feedback
 ja-jp/lp
-japan/directx/default.asp?
-japan/enable/textview.asp?
-japan/products/library/search.asp?
-japan/showcase/print/default.aspx?
-japan/terminology/query.asp?
 job-apply
 jobs-guest
 jobs/EU_UpdateJobEmployerCounter
@@ -3077,9 +2958,7 @@ m/?
 m/finance
 m=1$
 m?
-mac/help.mspx?
 macros
-mactopia/help.mspx?
 mail/$
 mail?
 mailinglist
@@ -3211,7 +3090,6 @@ min=
 mini.php
 missingpage
 mld?
-mnp_utility.mspx?
 mob/tracking
 mobile
 mobile/cities-transformed
@@ -3234,7 +3112,6 @@ mtb_composer
 multiproduct
 muse/identity/v2/identity.js
 music
-music/*/wal/
 music/album
 music/artist
 musica
@@ -3763,7 +3640,6 @@ register
 registration
 registry/wishlist
 remotelogout
-rep=hc
 report
 report-abuse?report=
 report=
@@ -3792,8 +3668,6 @@ row
 rss
 rss/people/*/reviews
 ru-ru.www.cloudflare.com
-ru-ru/events/platforma/materials/default.aspx?speaker*
-rus/licensing/Unilateral.aspx/*
 rw/WashingtonPost/Content/Epaper
 rweb
 s
@@ -3986,7 +3860,7 @@ shopping/reviewer
 shopping/seller
 shopping/suppliers/search
 shopping?
-showtimes/location/*http://patreon.com/robots.txt
+showtimes/location/*http://tripadvisor.com/robots.txt
 sidewiki/entry
 signature
 signin/inject
@@ -4277,8 +4151,6 @@ sounds/search$
 sounds/search?
 source=
 source_repo=
-spain/empresas/
-spain/medianaempresa/
 specialoffers
 specials
 spice
@@ -5052,13 +4924,6 @@ wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה
 wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים
 wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים
 wiki/ויקיפדיה:רשימת_מועמדים_למחיקה
-windows/404.aspx?*
-windows/campaign/features-built-in-apps.aspx
-windows/campaign/meet-apps.aspx
-windows/campaign/meet-start.aspx
-windows/compatibility/windows-7/
-windows/compatibility/windows-7/*
-windows/compatibility/windows-vista/
 wirecutter/*&xid=
 wirecutter/*.csv$
 wirecutter/*.zip$

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt

@@ -3069,7 +3069,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /newssearch
 /no-internet-heading-assigned
 /no-internet-heading-assisted
-/no-referrer-when-downgradehttp://video.twimg.com/robots.txt
+/no-referrer-when-downgradehttp://gmail.com/robots.txt
 /no-search-results
 /node
 /node/add
@@ -3125,6 +3125,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /password
 /patches/new
 /path_locks
+/patterns/
 /pay-invoice
 /pd/Ghost-Stories-Original-Tales-Ghostly-Urban-Legends-Audiobook/B07GL5B67Q
 /pd/ITEM_NAME-Podcast
@@ -3312,6 +3313,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /search/*?q=
 /search/?*terms
 /search/?q
+/search/?q=*
 /search/account
 /search/ai-search
 /search/collections
@@ -3347,6 +3349,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /search?searchProvider=*&sort=
 /search?title=
 /searchDirectory
+/searchresults/
 /searchsuggest
 /secure/
 /select*?
@@ -3624,7 +3627,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*__rmid
 ?*__rmidpage
 ?*ajs_event
-?*apikey*http://lifestylesimplify.com/robots.txt
+?*apikey*http://visitazores.com/robots.txt
 ?*app
 ?*appidt=*
 ?*aspects=
@@ -3844,7 +3847,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*utm_medium
 ?*utm_source
 ?*utm_source=
-?*utm_source=http://cricketarchive.com/robots.txt
+?*utm_source=http://chrome.google.com/robots.txt
 ?*verified=
 ?*xid
 ?252525253Bpreview=
@@ -5537,6 +5540,7 @@ HotelsManageListing
 HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
+HpImageArchive.aspx
 HubStyle/static-2.309
 HubStyle/static-2.313
 HubStyle/static-2.315
@@ -6095,6 +6099,7 @@ Project
 PromoTools
 PromotionDetailsDisplayView
 PromotionDisplayView
+Proxy.ashx
 ProxyToolsAccountSettingsTool
 ProxyToolsBillingAndPaymentsTool
 ProxyToolsBillingDetailTool
@@ -6227,6 +6232,7 @@ Saves
 ScreensaverShade
 Scripts
 Search
+Search
 Search/*
 Search/Mail/Public
 SearchNavigationView
@@ -6833,6 +6839,8 @@ ac/
 ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search
 ac2
 academic-subscriptions
+academic/profile
+academic/search
 academy
 academy-free-account
 academy/*/results
@@ -6855,7 +6863,7 @@ account-settings
 account-setup
 account/
 account/accomplishments
-account/accomplishments/http://glassdoor.com/robots.txt
+account/accomplishments/http://creativecommons.org/robots.txt
 account/ackgift
 account/deactivated
 account/emailoptout/?*token=
@@ -6916,6 +6924,7 @@ ach/search
 ach/user
 ach/users
 achievements
+aclick
 aclk
 acoustic
 acq/nonprofits
@@ -7178,6 +7187,7 @@ alexa-top-sites/*
 alexa-web-search/*
 alexa/*
 alexaforbusiness/
+alink
 alkali
 all
 all-supported-params
@@ -7204,6 +7214,7 @@ amp
 amp$
 amp-proxy
 amp-tealium
+amp/
 amphtml
 amphtml/blogs/test
 amphtml/classic-apps
@@ -7264,16 +7275,17 @@ api.html?
 api.php
 api/
 api/?
-api/attachments/*http://euronews.com/robots.txt
+api/attachments/*http://docs.wixstatic.com/robots.txt
 api/discovery
 api/editionurlgenerator
 api/eventlogging
 api/html
-api/http://globenewswire.com/robots.txt
+api/http://kobo.com/robots.txt
 api/internal/*/location
 api/internal/analytics/songs
 api/jobPostings/jobs
 api/legendary-data/relay
+api/maps/
 api/nextsteps/*
 api/nicorepo/timeline
 api/partner_select/*/price_query
@@ -7342,7 +7354,7 @@ app_data/
 appchart
 appealed-pin
 apple-app-site-association
-apple-app-site-associationhttp://nicovideo.jp/robots.txt
+apple-app-site-associationhttp://fonts.google.com/robots.txt
 apple-news
 appliance
 applicant-eligibility
@@ -8393,6 +8405,7 @@ bezahlvorgaenge.html
 bfe/apps/website
 bfl
 bfmp
+bfp/search
 bg
 bg-*/
 bg-bg/
@@ -8455,6 +8468,7 @@ bin
 bin/
 bin/search.go
 bin/search.py
+bing-site-safety
 bingbot
 bipolar-disorder/bipolar-depression-management
 bipolar-disorder/video/bipolar-fatherhood
@@ -8544,7 +8558,7 @@ blog/*?text=
 blog/*sitemap.xml$
 blog/*utm_source=
 blog/.
-blog/.html.*http://hbr.org/robots.txt
+blog/.html.*http://salon.com/robots.txt
 blog/?p=
 blog/?s=
 blog/Neuer_Beitrag.html
@@ -8600,6 +8614,7 @@ blogs/blogpost/preview
 blogs/nationals-journal-beta
 blogs/pad$
 blogs/pad/$
+blogs/search/
 blogs/slow-ride
 blogs/technology.atom
 blogs/test
@@ -8671,7 +8686,7 @@ bookmark_mana2
 bookmarklet
 bookmarks
 bookmarks/widget
-bookmarkshttp://whatsapp.com/robots.txt
+bookmarkshttp://ed.ted.com/robots.txt
 books
 books/NBK
 books/about
@@ -9478,7 +9493,7 @@ cdn-cgi
 cdn-cgi/
 cdn-cgi/bm/cv
 cdn-cgi/challenge-platform
-cdn-cgi/http://google.co.in/robots.txt
+cdn-cgi/http://mediapost.com/robots.txt
 cdn-cgi/rum
 cdn-cgi/zaraz/s.js
 cdn-cgi/zaraz/t
@@ -9505,7 +9520,7 @@ cgi-bin/homelandsecurity
 cgi-bin/loginpage
 cgi-bin/state
 cgi-bin/wx
-cgi-binhttp://youtube.de/robots.txt
+cgi-binhttp://ifttt.com/robots.txt
 cgi-local
 cgi-perl
 cgi-registry
@@ -9694,6 +9709,7 @@ civis/posts
 civis/search
 civis/threads/*in_iframe=1
 civis/threads/*wp_data=
+ck/
 cl
 cl2/feeds
 cl2/ical
@@ -9835,7 +9851,7 @@ co/users
 co/ventureone
 code
 code/*
-code/http://m.youtube.com/robots.txt
+code/http://w3techs.com/robots.txt
 codeCoverage
 codesamples/
 coin*mid=
@@ -9959,7 +9975,6 @@ compagnie
 companies/c
 companies/o
 companies/search
-company
 company/*.rss
 company/?s=
 company/add-ons-agreement
@@ -10045,7 +10060,7 @@ confirmation.html
 conflagexp
 connect
 connect/@
-connect/http://godaddy.com/robots.txt
+connect/http://blog.twitter.com/robots.txt
 connect/social-intranet.html
 connections
 connections/recent
@@ -10161,7 +10176,7 @@ content/dam/ircc/documents/pdf/francais/trousses/form/imm5476
 content/dam/ircc/documents/pdf/francais/trousses/form/imm5644
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0002
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0004
-content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://shareasale.com/robots.txt
+content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://canalplus.fr/robots.txt
 content/dam/ircc/documents/pdf/francais/trousses/guides/guide-0142-avion-afghanistan-voie-acces-rp.pdf
 content/dam/oecd/
 content/dam/redcross/lp
@@ -10348,10 +10363,12 @@ cpresources
 cpro
 cpspa
 cr
+cr$
 cr/
 cr/$
 cr/corporate/contact
 cr/news
+cr?
 craft
 crash
 crawler4j
@@ -10809,6 +10826,7 @@ deals/merchant/*
 deals/popular/*
 debats/
 debug
+debug/
 decision-makers$
 decision-makers/*/admin_accounts$
 decision-makers/*/admin_accounts?
@@ -10839,6 +10857,7 @@ delight
 dell/cities-transformed
 dell/cloud-future-of-medicine
 demo
+demokratie-jetzt.html
 demonstration
 demos
 demos/
@@ -10892,6 +10911,7 @@ dg-service
 dgs/education_culture/404_en.htm
 dhtml
 dhtmlmenu
+di-ag/
 diagnosed-with-glioblastoma
 dialog/*
 dictionary/static/sitemaps
@@ -11470,7 +11490,7 @@ email-preferences/
 email-unsubscribe
 email-validation
 email/$
-email/*http://moma.org/robots.txt
+email/*http://dw.com/robots.txt
 email/privacy-guarantees
 email/privacy-terms
 email/unsubscribe
@@ -11680,6 +11700,9 @@ enterprise/orgs-terms
 enterprise/seo
 entertainment/article/Minerva-s-horoscope
 entertainment/bravo
+entities/search
+entityexplore$
+entityexplore?
 entrez
 entrez/eutils
 entrez/sutils
@@ -12210,6 +12233,7 @@ fbl/terms.html
 fbpostaction.cms?article=
 fcgi-
 fckeditor
+fd/
 fdbk
 feature/sf-test
 feature_nine_browse-bin=
@@ -12420,7 +12444,7 @@ files/techabuse-welsh.pdf
 files/techabuse.pdf
 fileview
 filing/free-file-fillable-forms/free-file-fillable-forms-support
-filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://globalresearch.ca/robots.txt
+filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://is.gd/robots.txt
 filter
 filter/tips
 filter/tips
@@ -12814,6 +12838,8 @@ full
 fullscreen
 fullscreen/ReportLinker
 fun
+fun/api/
+fun/g/
 fundraiser/charity
 fundraiser/search
 fur/forums
@@ -12835,7 +12861,7 @@ fy-NL/search
 fy-NL/user
 fy-NL/users
 g
-g/4arE9Chttp://namecheap.com/robots.txt
+g/4arE9Chttp://flickr.com/robots.txt
 g/api
 g00
 g01
@@ -13065,7 +13091,7 @@ gh/news
 gh/useracquisition
 gh_jid=
 gif
-gifshttp://l.facebook.com/robots.txt
+gifshttp://gist.github.com/robots.txt
 gift-redemption
 gift-terms
 gift_card=
@@ -13092,6 +13118,7 @@ glass/images/commerce
 global
 global$
 global-consumer-survey/tool
+global-infrastructure/services/get-async/http://baidu.com/robots.txt
 global-master
 global-nav-esi
 global/choose-location.html?location=home
@@ -13422,7 +13449,7 @@ ha/search
 ha/user
 ha/users
 hackathons/reinvent-2013/
-hacker/*/followhttp://justice.gov/robots.txt
+hacker/*/followhttp://hackerone.com/robots.txt
 hacker/*/unfollow
 hair-transplant-services
 handler
@@ -13720,12 +13747,14 @@ hosted
 hosted/images
 hosting/search
 hotel/index.hi.
+hotel/reviews?
 hotel/us/the-airstream-van.*.html
 hotel_attractions
 hotel_rt_onview
 hotelfinder/rpc
 hotels
 hotels/rpc
+hotels/search
 hotelsonmap.*.json
 hotfix/
 houseads
@@ -13739,6 +13768,8 @@ howto
 howtobuy/
 hp/focus-retrocompatibile$
 hp/video
+hpm
+hpmob
 hr
 hr-*/
 hr-hr/
@@ -13819,37 +13850,38 @@ http
 http-checker
 http2_data
 http:
-http://ads.google.com/robots.txt
-http://adssettings.google.com/robots.txt
-http://airtable.com/robots.txt
-http://amazon.fr/robots.txt
-http://blacklivesmatter.com/robots.txt
-http://booking.com/robots.txt
-http://chicagotribune.com/robots.txt
-http://cofchrist.org/robots.txt
-http://fb.com/robots.txt
-http://focus.de/robots.txt
-http://g.co/robots.txt
-http://gmail.com/robots.txt
-http://google.com.br/robots.txt
-http://i2.wp.com/robots.txt
-http://idfa.nl/robots.txt
-http://img.youtube.com/robots.txt
-http://lapakbrebes.com/robots.txt
-http://livescience.com/robots.txt
-http://mediafire.com/robots.txt
-http://mobile.twitter.com/robots.txt
-http://orcd.co/robots.txt
-http://roguefitness.com/robots.txt
-http://s3-eu-west-1.amazonaws.com/robots.txt
-http://sfgate.com/robots.txt
-http://shopify.com/robots.txt
-http://tech.nikkeibp.co.jp/robots.txt
-http://thenextweb.com/robots.txt
-http://usnews.com/robots.txt
-http://visitazores.com/robots.txt
-http://weibo.com/robots.txt
-http://woodgears.ca/robots.txt
+http://2.bp.blogspot.com/robots.txt
+http://about.google/robots.txt
+http://amazon.de/robots.txt
+http://amzn.to/robots.txt
+http://api.whatsapp.com/robots.txt
+http://app.box.com/robots.txt
+http://bbc.com/robots.txt
+http://bcn.cl/robots.txt
+http://behance.net/robots.txt
+http://census.gov/robots.txt
+http://euronews.com/robots.txt
+http://extension.purdue.edu/robots.txt
+http://graphisoft.com/robots.txt
+http://hubspot.com/robots.txt
+http://ie.edu/robots.txt
+http://linkedin.com/robots.txt
+http://mrisoftware.com/robots.txt
+http://nationalgeographic.com/robots.txt
+http://news.yahoo.com/robots.txt
+http://patreon.com/robots.txt
+http://producthunt.com/robots.txt
+http://raiplay.it/robots.txt
+http://redbubble.com/robots.txt
+http://support.apple.com/robots.txt
+http://techradar.com/robots.txt
+http://theverge.com/robots.txt
+http://tiny.cc/robots.txt
+http://torproject.org/robots.txt
+http://translate.google.com/robots.txt
+http://upload.wikimedia.org/robots.txt
+http://vogue.com/robots.txt
+http://yelp.com/robots.txt
 hu
 hu-*/
 hu-hu/
@@ -14221,11 +14253,16 @@ images/$
 images/*
 images/?
 images/Podcasts
+images/async/?
+images/async?
 images/carBlogsimages
 images/chart_gifs
 images/chart_pdfs
 images/chart_png300
 images/chart_png460
+images/hosted/?
+images/hosted/search?
+images/hosted?
 images/interview_pdfs
 images/li.gif
 images/logging_requests.gif
@@ -14234,8 +14271,13 @@ images/ndtvvideo/*
 images/pad/search?rpt=imageview
 images/pad/search?text=*&source=morda
 images/resume
+images/search%3F
+images/search%5C
+images/search/?
+images/search?
 images/search?rpt=imageview
 images/search?text=*&source=morda
+images/searchbyimage
 images/showroomimages
 images/smart/$
 images/touch/$
@@ -14246,6 +14288,7 @@ images/usedcarimages
 images/userprofileimg
 images/v$
 images/viewconfig$
+images?
 imagine/
 imesync
 img
@@ -14368,6 +14411,7 @@ inapp
 inbox
 inbox$
 inbox$
+inboxcommerce
 inc5000dev
 include
 include/
@@ -14481,7 +14525,6 @@ info/jobs/Kooperations-Manager.html
 info/jobs/Social-Media-Online-Kommunikation.html
 info/jobs/UX-UI-Designer.html
 info/jobs/ba-medieninformatik.html
-info/jobs/backend-entwicklerin.html
 info/jobs/buchhaltung-payment.html
 info/jobs/buchhaltung-personal.html
 info/jobs/compliance.html
@@ -14494,6 +14537,7 @@ info/jobs/product-owner.html
 info/jobs/projektberater.html
 info/jobs/scrum-master.html
 info/jobs/showmaster.html
+info/jobs/site-reliability-engineer.html
 info/jobs/social-media.html
 info/jobs/support.html
 info/jobs/texterin-uxwriterin.html
@@ -14527,7 +14571,7 @@ intcmp=
 integrate/
 integromedb
 intelligencer/amp
-interactive*http://ffm.to/robots.txt
+interactive*http://fcc.gov/robots.txt
 interactive/reading-preference
 interface/pda
 intern/service/suche
@@ -15381,7 +15425,6 @@ latest-playlist
 latest-talk
 latin
 latin_en
-launch-day-show-voting.html
 launch2005/
 law-school/search-results/
 lawlib/archive
@@ -15406,7 +15449,7 @@ layout/set/txt_pq
 layout/set/xml
 layouts
 lb
-leadingtogether/http://click.linksynergy.com/robots.txt
+leadingtogether/http://validator.w3.org/robots.txt
 leaf
 learn
 learn-noperf
@@ -15636,6 +15679,7 @@ loans
 loc/volunteers
 loc=
 local
+local
 local-services-ads
 local/ajax
 local/blogsandcolumns/slow-ride-front
@@ -15950,6 +15994,7 @@ maps/*?lang=uk$
 maps/*?lang=uz$
 maps/*?ll=
 maps/about
+maps/adsendpoint
 maps/api/js
 maps/api/place/js
 maps/api/staticmap
@@ -16076,7 +16121,7 @@ media/tourism-content
 media/travelpod-roll.flv
 media/tv/channel/credits
 media/tyfo*
-media/tyfo*http://umc.org/robots.txt
+media/tyfo*http://statcounter.com/robots.txt
 media/unesco
 media/vr-geo-photo-j
 media/vr-geo-photo-l
@@ -16401,6 +16446,7 @@ merchant/directions/*
 merchant/imageUploadPreview/*
 merchant/loc/*
 merchant/moreProducts?*
+merchant/reviews?
 merchant/update/*
 merchantapps
 mes
@@ -17378,6 +17424,7 @@ notificarelacionadas
 notifications
 notifications
 notifications$
+notifications/
 notifications/*
 nova
 now/mt3
@@ -17562,6 +17609,8 @@ offers/nonprofits
 offers/nonprofits$
 offers/nonprofits%
 offers/nonprofits?
+offers/proxy/dealsserver/api/log
+offers/proxy/dealsserver/buy
 offers/referral-terms-conditions
 offers/referral-terms-conditions$
 offers/referral-terms-conditions%
@@ -17796,7 +17845,6 @@ page/*/?egotecpage
 page/*/?hc_location=
 page/*/?iq_id=
 page/*/?pcd=
-page/*/?s
 page/*/?s=
 page/*/?search=
 page/account-confirmation-thanks
@@ -18652,6 +18700,7 @@ pro-resources
 pro-site
 pro/about
 pro2
+proFile/history?
 proMatch
 proProjects
 proVideoWidget
@@ -18669,6 +18718,7 @@ product-reviews/B0069IY63Y
 product/electrode/api/logger
 product/electrode/api/wpa
 product/recommended
+product/reviews?
 productBrand=
 productId=
 productPurchaseDone
@@ -18720,6 +18770,7 @@ profile/*/followers
 profile/*/following
 profile/*/map$
 profile/downloads_limit
+profile/history?
 profile/license
 profile/payment/getstarted
 profile/view
@@ -19378,6 +19429,7 @@ responder
 rest
 rest/*/activity
 results
+results
 results-export-email-by-search-data
 results-export-ids
 results-export-search-by-year
@@ -19430,6 +19482,7 @@ reviews/seller
 reviews/top-reviewers
 reviews_page=
 rewards
+rewardsapp/
 rf=*
 ribbon.php
 rip/
@@ -20090,7 +20143,7 @@ section/3
 section/36
 section/6
 section/8
-section/9http://telegram.me/robots.txt
+section/9http://issuu.com/robots.txt
 sectionized
 sections
 sections$
@@ -20257,6 +20310,7 @@ set_cookie.pl
 setbucket
 setprefs
 settings
+settings
 settings/*
 settings/deactivated
 settings/linkaccount
@@ -20390,6 +20444,7 @@ sharing
 sharing/files
 sharing/folders
 shelf/user_shelves
+shenghuo
 shenghuo/search?
 shifen/
 shihui
@@ -20400,11 +20455,15 @@ shipping-payment
 shippingassistant
 shippinginformation
 shop
+shop$
+shop/
 shop/*&
 shop/browse/overlay
+shop/collections
 shop/global
 shop/iphone/payments/overlay
 shop/search.ep
+shop?
 shop_name_search_service
 shop_pins
 shopfront-api
@@ -20455,7 +20514,7 @@ showdependency
 shows/
 showtag
 showtime/.requestinvite.html.swp
-showtimes/location/*http://tripadvisor.com/robots.txt
+showtimes/location/*http://img.youtube.com/robots.txt
 shr*
 shutterstock
 si
@@ -20510,7 +20569,6 @@ sign-in
 sign-in-6for6Sale
 sign-out
 sign-out
-sign-up
 sign-up-to-be-contacted-for-research
 sign-up/?
 sign_in
@@ -20998,6 +21056,7 @@ social
 social-connect/send
 social/photosui/images
 social/photosui/sitemap
+social/search?
 social/social-security-directory/cai
 social_media
 socialapp/
@@ -21109,6 +21168,7 @@ sp/3for2
 sp=
 sp_url=
 spartan
+spbasic
 spbot
 spc-source=
 special-promo/2for1
@@ -21186,6 +21246,7 @@ sports/whatson
 sports_fp_ms
 spp/
 spreadsheet
+spresults
 sprint_wml
 sprint_xhtml
 sq/android/collections
@@ -21269,6 +21330,7 @@ statements
 static
 static-assets/documents/PwnedPasswordsTop100k.txt
 static-assets/images/collection
+static/
 static/codemirror
 static/css
 static/decodeKey.txt
@@ -21387,6 +21449,8 @@ storeclick
 storefront
 storepicker
 storerequest
+stores$
+stores/
 stores/page/*visitId=
 stories
 story
@@ -21685,7 +21749,7 @@ tag
 tag-manager
 tag-manager-360
 tag/
-tag/*+http://drupal.org/robots.txt
+tag/*+http://discogs.com/robots.txt
 tag/*/a
 tag/*/a/
 tag/*/b
@@ -21912,7 +21976,7 @@ test5
 test6
 testcases
 testfl
-testhttp://deepmind.com/robots.txt
+testhttp://moz.com/robots.txt
 testing
 testpage-forhomepage
 tests/*
@@ -21921,6 +21985,7 @@ text
 text_edition
 tfemail
 th
+th$
 th-*/
 th-th.www.cloudflare.com
 th-th/
@@ -21945,6 +22010,7 @@ th/solutions/case-studies/innovators/innovators-eu/
 th/solutions/case-studies/sustainability-emea/
 th/user
 th/users
+th?
 th_th/templates/M
 thank-you
 thank-you-enterprise-demo
@@ -22193,12 +22259,13 @@ translate_static
 translate_suggestion
 translate_t
 translation
+translator/?ref=
 transmute/static-1.85
 transmute/static-2.6
 transmute/static-2.8
 transmute/static-2.9
 transparency
-transparency/http://atlasobscura.com/robots.txt
+transparency/http://anchor.fm/robots.txt
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14885
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14895
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14896
@@ -22221,14 +22288,28 @@ travel/
 travel/2013/aug/22/been-there-readers-competition?
 travel/8711559/The-Telegraph-Travel-Awards-2011.html
 travel/clk
+travel/css
 travel/entity
+travel/flight/flightSearch
+travel/flight/flightSearchAction
+travel/flight/search/?
+travel/flight/search?
 travel/flights/s
+travel/hotel-search
 travel/hotel/e
+travel/hotel/hotelMiniSearchRequest
+travel/hotel/hotelSearch
+travel/hotels/checkout
+travel/hotels/detail
 travel/hotels/entity
 travel/hotels/hotel-finder
 travel/hotels/hotel-finder/$
+travel/hotels/search/?
+travel/hotels/search?
 travel/hotels/stories
+travel/scripts
 travel/search
+travel/secure
 travelnevadahighway95tc
 treasury
 treasury/treasdocuments
@@ -22727,6 +22808,7 @@ ur/search
 ur/user
 ur/users
 urchin_test
+url
 urlinator/static-1.54
 urlinator/static-1.55
 urlinator/static-1.57
@@ -23245,14 +23327,29 @@ video_file
 videodash.cms*
 videorequest
 videos
+videos/?
 videos/assets
+videos/browsing
 videos/e
+videos/explore
+videos/favorites
+videos/feed
+videos/feedcontent
 videos/get_video_by_uid/
 videos/images
 videos/js
+videos/music
+videos/music/
+videos/music/?
+videos/music?
 videos/search.php
+videos/search/?
+videos/search?
+videos/trending
+videos/trendingcontent
 videos/viewmeta/
 videos/week
+videos?
 videosearch
 view
 viewReview
@@ -23458,6 +23555,10 @@ whythisad
 widget
 widget-landing-page.html
 widget/*
+widget/cr
+widget/entity/search/?
+widget/render
+widget/snapshot
 widget_auth.php
 widget_chunks
 widgets
@@ -24233,6 +24334,7 @@ word/document_url
 word/download
 word/embed
 work
+work
 work/_
 work/acceptinvitation
 work/account
@@ -24330,9 +24432,9 @@ wp-admin
 wp-admin/
 wp-admin/admin-ajax.php
 wp-admin/admin-ajax.php
+wp-admin/admin-ajax.phphttp://aljazeera.com/robots.txt
 wp-admin/admin-ajax.phphttp://fool.com/robots.txt
-wp-admin/admin-ajax.phphttp://today.com/robots.txt
-wp-admin/admin-ajax.phphttp://usps.com/robots.txt
+wp-admin/admin-ajax.phphttp://openstreetmap.org/robots.txt
 wp-admin/load-scripts.php
 wp-admin/load-styles.php
 wp-adv
@@ -24402,7 +24504,7 @@ wp-content/uploads/
 wp-content/uploads/wc-logs
 wp-content/uploads/woocommerce_transient_files
 wp-content/uploads/woocommerce_uploads
-wp-content/uploads/wp-import-export-lite/http://androidcommunity.com/robots.txt
+wp-content/uploads/wp-import-export-lite/http://rebrand.ly/robots.txt
 wp-dyn
 wp-includes
 wp-includes*.css
@@ -24527,7 +24629,7 @@ your-account
 your-orders
 your-shop
 youreurope/businessACC
-youreurope/businessDEV/http://business.facebook.com/robots.txt
+youreurope/businessDEV/http://europarl.europa.eu/robots.txt
 youreurope/citizensACC
 youreurope/citizensDEV
 yours
@@ -24877,7 +24979,7 @@ zz
 8618
 9742
 9744
-101612.rsshttp://sustainability.google/robots.txt
+101612.rsshttp://c-span.org/robots.txt
 1010203
 1014943
 1027454