Kundenlogin

Warum kann mein Bot/Crawler meine Webseite nicht aufrufen?

  • FAQ
  • Hosting
  • Warum kann mein Bot/Crawler meine Webseite nicht aufrufen?

Mögliches Problem

  • Bestimmte Bots/Crawler können Ihre Webseite nicht mehr besuchen.
  • Die Anfrage des Bots wird mit einem „Error 501“ beantwortet.

 

Mögliche Ursache

Im März 2017 haben wir für alle Webhosting, Reseller und Server Produkte mit Plesk eine Reihe von neuen Mod_Security Regeln verteilt.

 

Nähere Informationen zum Hintergrund finden Sie in unserem Blogbeitrag „Beta: Sperrliste für aggressive Bots„.

 

Mögliche Lösung

Jeder Bot lässt sich einzeln pro Domain wieder freigeben.

 

Fügen Sie dazu folgende Anweisungen in das Feld „Zusätzliche Anweisungen für HTTP und HTTPS“ der jeweiligen Domain ein.

Plesk: Zusätzliche Apache-Anweisungen Bots

Copy & Paste

<IfModule mod_security2.c>
SecRuleRemoveById 400235
SecRuleRemoveById 400234
</IfModule>

 

Liste der Bots und IDs

"Ezooms" = id:400036
"Spiderlytics" = id:400037
"BLEXBot" = id:400038
"MJ12bot" = id:400039
"AhrefsBot" = id:400040
"Aboundex" = id:400041
"80legs" = id:400042
"360Spider" = id:400043
"Cogentbot" = id:400044
"Alexibot" = id:400045
"asterias" = id:400046
"attach" = id:400047
"BackDoorBot" = id:400048
"BackWeb" = id:400049
"Bandit" = id:400050
"BatchFTP" = id:400051
"Bigfoot" = id:400052
"Black.Hole" = id:400053
"BlackWidow" = id:400054
"BlowFish" = id:400055
"BotALot" = id:400056
"Buddy" = id:400057
"BuiltBotTough" = id:400058
"Bullseye" = id:400059
"BunnySlippers" = id:400060
"Cegbfeieh" = id:400061
"CheeseBot" = id:400062
"CherryPicker" = id:400063
"ChinaClaw" = id:400064
"Collector" = id:400065
"Copier" = id:400066
"CopyRightCheck" = id:400067
"cosmos" = id:400068
"Crescent" = id:400069
"Custo" = id:400070
"AIBOT" = id:400071
"DISCo" = id:400072
"DIIbot" = id:400073
"DittoSpyder" = id:400074
"dragonfly" = id:400075
"Drip" = id:400076
"eCatch" = id:400077
"EasyDL" = id:400078
"ebingbong" = id:400079
"EirGrabber" = id:400080
"EmailCollector" = id:400081
"EmailSiphon" = id:400082
"EmailWolf" = id:400083
"EroCrawler" = id:400084
"Exabot" = id:400085
"Express" = id:400086
"Extractor" = id:400087
"EyeNetIE" = id:400088
"Foobot" = id:400089
"flunky" = id:400090
"FrontPage" = id:400091
"Go-Ahead-Got-It" = id:400092
"gotit" = id:400093
"GrabNet" = id:400094
"Grafula" = id:400095
"Harvest" = id:400096
"hloader" = id:400097
"HMView" = id:400098
"HTTrack" = id:400099
"humanlinks" = id:400100
"IlseBot" = id:400101
"InfoNaviRobot" = id:400102
"InfoTekies" = id:400103
"Intelliseek" = id:400104
"InterGET" = id:400105
"Internet" = id:400106
"Jakarta" = id:400107
"JennyBot" = id:400108
"JetCar" = id:400109
"JustView" = id:400110
"Jyxobot" = id:400111
"Kenjin.Spider" = id:400112
"Keyword.Density" = id:400113
"larbin" = id:400114
"LexiBot" = id:400115
"lftp" = id:400116
"libWeb/clsHTTP" = id:400117
"likse" = id:400118
"LinkextractorPro" = id:400119
"LinkScan/8.1a.Unix" = id:400120
"LNSpiderguy" = id:400121
"LinkWalker" = id:400122
"lwp-trivial" = id:400123
"LWP::Simple" = id:400124
"Magnet" = id:400125
"Mag-Net" = id:400126
"MarkWatch" = id:400127
"Mass" = id:400128
"Mata.Hari" = id:400129
"Memo" = id:400130
"MIDown" = id:400131
"MIIxpc" = id:400132
"Mirror" = id:400133
"Missigua" = id:400134
"Mister" = id:400135
"moget" = id:400136
"NAMEPROTECT" = id:400137
"Navroad" = id:400138
"NearSite" = id:400139
"NetAnts" = id:400140
"Netcraft" = id:400141
"NetMechanic" = id:400142
"NetSpider" = id:400143
"NetZIP" = id:400144
"NextGenSearchBot" = id:400145
"NICErsPRO" = id:400146
"niki-bot" = id:400147
"NimbleCrawler" = id:400148
"Ninja" = id:400149
"NPbot" = id:400150
"Octopus" = id:400151
"Offline" = id:400152
"Openfind" = id:400153
"OutfoxBot" = id:400154
"PageGrabber" = id:400155
"Papa" = id:400156
"pavuk" = id:400157
"pcBrowser" = id:400158
"Pockey" = id:400159
"ProPowerBot/2.14" = id:400160
"ProWebWalker" = id:400161
"psbot" = id:400162
"QueryN.Metasearch" = id:400163
"RealDownload" = id:400164
"Reaper" = id:400165
"Recorder" = id:400166
"ReGet" = id:400167
"RepoMonkey" = id:400168
"Siphon" = id:400169
"SiteSnagger" = id:400170
"SlySearch" = id:400171
"SmartDownload" = id:400172
"Snake" = id:400173
"Snapbot" = id:400174
"Snoopy" = id:400175
"sogou" = id:400176
"SpaceBison" = id:400177
"SpankBot" = id:400178
"spanner" = id:400179
"Sqworm" = id:400180
"Stripper" = id:400181
"Sucker" = id:400182
"SuperBot" = id:400183
"SuperHTTP" = id:400184
"Surfbot" = id:400185
"suzuran" = id:400186
"Szukacz/1.4" = id:400187
"tAkeOut" = id:400188
"Teleport" = id:400189
"Telesoft" = id:400190
"TurnitinBot/1.5" = id:400191
"The.Intraformant" = id:400192
"TheNomad" = id:400193
"TightTwatBot" = id:400194
"Titan" = id:400195
"True_Robot" = id:400196
"turingos" = id:400197
"TurnitinBot" = id:400198
"URLy.Warning" = id:400199
"Vacuum" = id:400200
"VCI" = id:400201
"VoidEYE" = id:400202
"WebAuto" = id:400203
"WebBandit" = id:400204
"Webclipping.com" = id:400205
"WebCopier" = id:400206
"WebEMailExtrac.*" = id:400207
"WebEnhancer" = id:400208
"WebFetch" = id:400209
"WebGo" = id:400210
"Web.Image.Collector" = id:400211
"WebLeacher" = id:400212
"WebmasterWorldForumBot" = id:400213
"WebReaper" = id:400214
"WebSauger" = id:400215
"Website" = id:400216
"Webster" = id:400217
"WebStripper" = id:400218
"WebWhacker" = id:400219
"WebZIP" = id:400220
"Whacker" = id:400221
"Widow" = id:400222
"WISENutbot" = id:400223
"WWWOFFLE" = id:400224
"WWW-Collector-E" = id:400225
"Xaldon" = id:400226
"Xenu" = id:400227
"Zeus" = id:400228
"ZmEu" = id:400229
"Zyborg" = id:400230
"SemrushBot" = id:400231
"Acunetix" = id:400232
"FHscan" = id:400233
"Baiduspider" = id:400234
"Yandex" = id:400235
Schlagworte:
War der Artikel hilfreich?